You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by jo...@apache.org on 2015/08/15 19:12:50 UTC
[01/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Repository: nifi
Updated Branches:
refs/heads/develop 6fc52a4e9 -> aa9988478
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ClusterDTO.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ClusterDTO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ClusterDTO.java
new file mode 100644
index 0000000..66e19d7
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ClusterDTO.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.api.dto;
+
+import com.wordnik.swagger.annotations.ApiModelProperty;
+import java.util.Collection;
+import java.util.Date;
+
+import javax.xml.bind.annotation.XmlType;
+import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
+import org.apache.nifi.web.api.dto.util.TimeAdapter;
+
+/**
+ * Details about the composition of the cluster at a specific date/time.
+ */
+@XmlType(name = "cluster")
+public class ClusterDTO {
+
+ private Collection<NodeDTO> nodes;
+ private Date generated;
+
+ /**
+ * @return collection of the node DTOs
+ */
+ @ApiModelProperty(
+ value = "The collection of nodes that are part of the cluster."
+ )
+ public Collection<NodeDTO> getNodes() {
+ return nodes;
+ }
+
+ public void setNodes(Collection<NodeDTO> nodes) {
+ this.nodes = nodes;
+ }
+
+ /**
+ * @return the date/time that this report was generated
+ */
+ @XmlJavaTypeAdapter(TimeAdapter.class)
+ @ApiModelProperty(
+ value = "The timestamp the report was generated."
+ )
+ public Date getGenerated() {
+ return generated;
+ }
+
+ public void setGenerated(Date generated) {
+ this.generated = generated;
+ }
+}
[25/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/SocketUtils.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/SocketUtils.java b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/SocketUtils.java
new file mode 100644
index 0000000..27d676a
--- /dev/null
+++ b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/SocketUtils.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.io.socket;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.security.KeyManagementException;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.UnrecoverableKeyException;
+import java.security.cert.CertificateException;
+
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLServerSocket;
+import javax.net.ssl.SSLSocket;
+
+import org.apache.nifi.logging.NiFiLog;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public final class SocketUtils {
+
+ private static final Logger logger = new NiFiLog(LoggerFactory.getLogger(SocketUtils.class));
+
+ public static Socket createSocket(final InetSocketAddress address, final SocketConfiguration config) throws IOException {
+ if (address == null) {
+ throw new IllegalArgumentException("Socket address may not be null.");
+ } else if (config == null) {
+ throw new IllegalArgumentException("Configuration may not be null.");
+ }
+
+ final Socket socket;
+
+ final SSLContext sslContext;
+ try {
+ sslContext = config.createSSLContext();
+ } catch (final Exception e) {
+ throw new IOException("Could not create SSLContext", e);
+ }
+
+ if (sslContext == null) {
+ socket = new Socket(address.getHostName(), address.getPort());
+ } else {
+ socket = sslContext.getSocketFactory().createSocket(address.getHostName(), address.getPort());
+ }
+
+ if (config.getSocketTimeout() != null) {
+ socket.setSoTimeout(config.getSocketTimeout());
+ }
+
+ if (config.getReuseAddress() != null) {
+ socket.setReuseAddress(config.getReuseAddress());
+ }
+
+ if (config.getReceiveBufferSize() != null) {
+ socket.setReceiveBufferSize(config.getReceiveBufferSize());
+ }
+
+ if (config.getSendBufferSize() != null) {
+ socket.setSendBufferSize(config.getSendBufferSize());
+ }
+
+ if (config.getTrafficClass() != null) {
+ socket.setTrafficClass(config.getTrafficClass());
+ }
+
+ if (config.getKeepAlive() != null) {
+ socket.setKeepAlive(config.getKeepAlive());
+ }
+
+ if (config.getOobInline() != null) {
+ socket.setOOBInline(config.getOobInline());
+ }
+
+ if (config.getTcpNoDelay() != null) {
+ socket.setTcpNoDelay(config.getTcpNoDelay());
+ }
+
+ return socket;
+ }
+
+ public static ServerSocket createServerSocket(final int port, final ServerSocketConfiguration config)
+ throws IOException, KeyManagementException, UnrecoverableKeyException, NoSuchAlgorithmException, KeyStoreException, CertificateException {
+ if (config == null) {
+ throw new NullPointerException("Configuration may not be null.");
+ }
+
+ final SSLContext sslContext = config.createSSLContext();
+ final ServerSocket serverSocket;
+ if (sslContext == null) {
+ serverSocket = new ServerSocket(port);
+ } else {
+ serverSocket = sslContext.getServerSocketFactory().createServerSocket(port);
+ ((SSLServerSocket) serverSocket).setNeedClientAuth(config.getNeedClientAuth());
+ }
+
+ if (config.getSocketTimeout() != null) {
+ serverSocket.setSoTimeout(config.getSocketTimeout());
+ }
+
+ if (config.getReuseAddress() != null) {
+ serverSocket.setReuseAddress(config.getReuseAddress());
+ }
+
+ if (config.getReceiveBufferSize() != null) {
+ serverSocket.setReceiveBufferSize(config.getReceiveBufferSize());
+ }
+
+ return serverSocket;
+ }
+
+ public static void closeQuietly(final Socket socket) {
+ if (socket == null) {
+ return;
+ }
+
+ try {
+ try {
+ // can't shudown input/output individually with secure sockets
+ if ((socket instanceof SSLSocket) == false) {
+ if (socket.isInputShutdown() == false) {
+ socket.shutdownInput();
+ }
+ if (socket.isOutputShutdown() == false) {
+ socket.shutdownOutput();
+ }
+ }
+ } finally {
+ if (socket.isClosed() == false) {
+ socket.close();
+ }
+ }
+ } catch (final Exception ex) {
+ logger.debug("Failed to close socket due to: " + ex, ex);
+ }
+ }
+
+ public static void closeQuietly(final ServerSocket serverSocket) {
+ if (serverSocket == null) {
+ return;
+ }
+
+ try {
+ serverSocket.close();
+ } catch (final Exception ex) {
+ logger.debug("Failed to close server socket due to: " + ex, ex);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/DiscoverableService.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/DiscoverableService.java b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/DiscoverableService.java
new file mode 100644
index 0000000..fc817e9
--- /dev/null
+++ b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/DiscoverableService.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.io.socket.multicast;
+
+import java.net.InetSocketAddress;
+
+/**
+ * A service that may be discovered at runtime. A service is defined as having a
+ * unique case-sensitive service name and a socket address where it is
+ * available.
+ *
+ */
+public interface DiscoverableService {
+
+ /**
+ * The service's name. Two services are considered equal if they have the
+ * same case sensitive service name.
+ *
+ * @return the service's name
+ */
+ String getServiceName();
+
+ /**
+ * @return the service's address
+ */
+ InetSocketAddress getServiceAddress();
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/DiscoverableServiceImpl.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/DiscoverableServiceImpl.java b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/DiscoverableServiceImpl.java
new file mode 100644
index 0000000..3737e95
--- /dev/null
+++ b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/DiscoverableServiceImpl.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.io.socket.multicast;
+
+import java.net.InetSocketAddress;
+import org.apache.commons.lang3.StringUtils;
+
+/**
+ * A basic implementation of the DiscoverableService interface. To services are
+ * considered equal if they have the same case-sensitive service name.
+ *
+ */
+public class DiscoverableServiceImpl implements DiscoverableService {
+
+ private final String serviceName;
+
+ private final InetSocketAddress serviceAddress;
+
+ public DiscoverableServiceImpl(final String serviceName, final InetSocketAddress serviceAddress) {
+ if (StringUtils.isBlank(serviceName)) {
+ throw new IllegalArgumentException("Service name may not be null or empty.");
+ } else if (serviceAddress == null) {
+ throw new IllegalArgumentException("Service address may not be null.");
+ }
+ this.serviceName = serviceName;
+ this.serviceAddress = serviceAddress;
+ }
+
+ @Override
+ public InetSocketAddress getServiceAddress() {
+ return serviceAddress;
+ }
+
+ @Override
+ public String getServiceName() {
+ return serviceName;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("[Discoverable Service: %s available at %s:%d]", serviceName, serviceAddress.getHostName(), serviceAddress.getPort());
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null) {
+ return false;
+ }
+ if (!(obj instanceof DiscoverableService)) {
+ return false;
+ }
+ final DiscoverableService other = (DiscoverableService) obj;
+ return !((this.serviceName == null) ? (other.getServiceName() != null) : !this.serviceName.equals(other.getServiceName()));
+ }
+
+ @Override
+ public int hashCode() {
+ int hash = 5;
+ hash = 53 * hash + (this.serviceName != null ? this.serviceName.hashCode() : 0);
+ return hash;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/MulticastConfiguration.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/MulticastConfiguration.java b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/MulticastConfiguration.java
new file mode 100644
index 0000000..d1c2156
--- /dev/null
+++ b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/MulticastConfiguration.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.io.socket.multicast;
+
+/**
+ */
+public final class MulticastConfiguration {
+
+ private MulticastTimeToLive ttl = DEFAULT_MULTICAST_TTL;
+
+ private Integer socketTimeout;
+
+ private Integer receiveBufferSize;
+
+ private Integer sendBufferSize;
+
+ private Boolean reuseAddress;
+
+ private Integer trafficClass;
+
+ private Boolean loopbackMode;
+
+ public static final MulticastTimeToLive DEFAULT_MULTICAST_TTL = MulticastTimeToLive.SAME_SUBNET;
+
+ public MulticastTimeToLive getTtl() {
+ return ttl;
+ }
+
+ public void setTtl(final MulticastTimeToLive ttl) {
+ if (ttl == null) {
+ throw new NullPointerException("Multicast TTL may not be null.");
+ }
+ this.ttl = ttl;
+ }
+
+ public Integer getSocketTimeout() {
+ return socketTimeout;
+ }
+
+ public void setSocketTimeout(Integer socketTimeout) {
+ this.socketTimeout = socketTimeout;
+ }
+
+ public Boolean getReuseAddress() {
+ return reuseAddress;
+ }
+
+ public void setReuseAddress(Boolean reuseAddress) {
+ this.reuseAddress = reuseAddress;
+ }
+
+ public Integer getReceiveBufferSize() {
+ return receiveBufferSize;
+ }
+
+ public void setReceiveBufferSize(Integer receiveBufferSize) {
+ this.receiveBufferSize = receiveBufferSize;
+ }
+
+ public Integer getSendBufferSize() {
+ return sendBufferSize;
+ }
+
+ public void setSendBufferSize(Integer sendBufferSize) {
+ this.sendBufferSize = sendBufferSize;
+ }
+
+ public Integer getTrafficClass() {
+ return trafficClass;
+ }
+
+ public void setTrafficClass(Integer trafficClass) {
+ this.trafficClass = trafficClass;
+ }
+
+ public Boolean getLoopbackMode() {
+ return loopbackMode;
+ }
+
+ public void setLoopbackMode(Boolean loopbackMode) {
+ this.loopbackMode = loopbackMode;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/MulticastListener.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/MulticastListener.java b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/MulticastListener.java
new file mode 100644
index 0000000..1ce2ea0
--- /dev/null
+++ b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/MulticastListener.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.io.socket.multicast;
+
+import java.io.IOException;
+import java.net.DatagramPacket;
+import java.net.InetSocketAddress;
+import java.net.MulticastSocket;
+import java.net.SocketException;
+import java.net.SocketTimeoutException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Implements a listener for protocol messages sent over multicast. If a message
+ * is of type MulticastProtocolMessage, then the underlying protocol message is
+ * passed to the handler. If the receiving handler produces a message response,
+ * then the message is wrapped with a MulticastProtocolMessage before being sent
+ * to the originator.
+ *
+ */
+public abstract class MulticastListener {
+
+ // constants
+ private static final int DEFAULT_SHUTDOWN_LISTENER_SECONDS = 5;
+ private static final int DEFAULT_MAX_PACKET_SIZE_BYTES = 512;
+
+ private static final Logger logger = new org.apache.nifi.logging.NiFiLog(LoggerFactory.getLogger(MulticastListener.class));
+
+ // immutable members
+ private final int numThreads;
+ private final InetSocketAddress multicastAddress;
+ private final MulticastConfiguration configuration;
+
+ private volatile ExecutorService executorService; // volatile to guarantee most current value is visible
+ private volatile MulticastSocket multicastSocket; // volatile to guarantee most current value is visible
+
+ private int shutdownListenerSeconds = DEFAULT_SHUTDOWN_LISTENER_SECONDS;
+ private int maxPacketSizeBytes = DEFAULT_MAX_PACKET_SIZE_BYTES;
+
+ public MulticastListener(
+ final int numThreads,
+ final InetSocketAddress multicastAddress,
+ final MulticastConfiguration configuration) {
+
+ if (numThreads <= 0) {
+ throw new IllegalArgumentException("Number of threads may not be less than or equal to zero.");
+ } else if (multicastAddress == null) {
+ throw new IllegalArgumentException("Multicast address may not be null.");
+ } else if (multicastAddress.getAddress().isMulticastAddress() == false) {
+ throw new IllegalArgumentException("Multicast group must be a Class D address.");
+ } else if (configuration == null) {
+ throw new IllegalArgumentException("Multicast configuration may not be null.");
+ }
+
+ this.numThreads = numThreads;
+ this.multicastAddress = multicastAddress;
+ this.configuration = configuration;
+ }
+
+ /**
+ * Implements the action to perform when a new datagram is received. This
+ * class must not close the multicast socket.
+ *
+ * @param multicastSocket socket
+ * @param packet the datagram packet
+ */
+ public abstract void dispatchRequest(final MulticastSocket multicastSocket, final DatagramPacket packet);
+
+ public void start() throws IOException {
+
+ if (isRunning()) {
+ return;
+ }
+
+ multicastSocket = MulticastUtils.createMulticastSocket(multicastAddress.getPort(), configuration);
+ multicastSocket.joinGroup(multicastAddress.getAddress());
+
+ executorService = Executors.newFixedThreadPool(numThreads);
+
+ final ExecutorService runnableExecServiceRef = executorService;
+ final MulticastSocket runnableMulticastSocketRef = multicastSocket;
+
+ new Thread(new Runnable() {
+ @Override
+ public void run() {
+ while (runnableExecServiceRef.isShutdown() == false) {
+ try {
+ final byte[] buf = new byte[maxPacketSizeBytes];
+ final DatagramPacket packet = new DatagramPacket(buf, maxPacketSizeBytes);
+ runnableMulticastSocketRef.receive(packet);
+ runnableExecServiceRef.execute(new Runnable() {
+ @Override
+ public void run() {
+ dispatchRequest(multicastSocket, packet);
+ }
+ });
+ } catch (final SocketException | SocketTimeoutException ste) {
+ /* ignore so that we can accept connections in approximately a non-blocking fashion */
+ } catch (final Exception e) {
+ logger.warn("Cluster protocol receiver encountered exception: " + e, e);
+ }
+ }
+ }
+ }).start();
+ }
+
+ public boolean isRunning() {
+ return (executorService != null && executorService.isShutdown() == false);
+ }
+
+ public void stop() throws IOException {
+
+ if (isRunning() == false) {
+ return;
+ }
+
+ // shutdown executor service
+ try {
+ if (getShutdownListenerSeconds() <= 0) {
+ executorService.shutdownNow();
+ } else {
+ executorService.shutdown();
+ }
+ executorService.awaitTermination(getShutdownListenerSeconds(), TimeUnit.SECONDS);
+ } catch (final InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ } finally {
+ if (executorService.isTerminated()) {
+ logger.info("Multicast Listener has been terminated successfully.");
+ } else {
+ logger.warn("Multicast Listener has not terminated properly. There exists an uninterruptable thread that will take an indeterminate amount of time to stop.");
+ }
+ }
+
+ // shutdown server socket
+ if (multicastSocket.isClosed() == false) {
+ multicastSocket.leaveGroup(multicastAddress.getAddress());
+ multicastSocket.close();
+ }
+
+ }
+
+ public int getShutdownListenerSeconds() {
+ return shutdownListenerSeconds;
+ }
+
+ public void setShutdownListenerSeconds(final int shutdownListenerSeconds) {
+ this.shutdownListenerSeconds = shutdownListenerSeconds;
+ }
+
+ public int getMaxPacketSizeBytes() {
+ return maxPacketSizeBytes;
+ }
+
+ public void setMaxPacketSizeBytes(int maxPacketSizeBytes) {
+ if (maxPacketSizeBytes <= 0) {
+ throw new IllegalArgumentException("Max packet size must be greater than zero bytes.");
+ }
+ this.maxPacketSizeBytes = maxPacketSizeBytes;
+ }
+
+ public MulticastConfiguration getConfiguration() {
+ return configuration;
+ }
+
+ public InetSocketAddress getMulticastAddress() {
+ return multicastAddress;
+ }
+
+ public int getNumThreads() {
+ return numThreads;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/MulticastServiceDiscovery.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/MulticastServiceDiscovery.java b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/MulticastServiceDiscovery.java
new file mode 100644
index 0000000..212e20c
--- /dev/null
+++ b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/MulticastServiceDiscovery.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.io.socket.multicast;
+
+import java.net.InetSocketAddress;
+
+/**
+ * Defines the interface for discovering services based on name. Services are
+ * expected to be exposed via socket address and port.
+ *
+ */
+public interface MulticastServiceDiscovery extends ServiceDiscovery {
+
+ /**
+ * @return the multicast address
+ */
+ InetSocketAddress getMulticastAddress();
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/MulticastServicesBroadcaster.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/MulticastServicesBroadcaster.java b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/MulticastServicesBroadcaster.java
new file mode 100644
index 0000000..7ef5a5d
--- /dev/null
+++ b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/MulticastServicesBroadcaster.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.io.socket.multicast;
+
+import java.net.InetSocketAddress;
+
+/**
+ * Defines the interface for broadcasting a service via multicast.
+ *
+ */
+public interface MulticastServicesBroadcaster extends ServicesBroadcaster {
+
+ /**
+ * @return the multicast address
+ */
+ InetSocketAddress getMulticastAddress();
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/MulticastTimeToLive.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/MulticastTimeToLive.java b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/MulticastTimeToLive.java
new file mode 100644
index 0000000..3e96c61
--- /dev/null
+++ b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/MulticastTimeToLive.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.io.socket.multicast;
+
+/**
+ */
+public enum MulticastTimeToLive {
+
+ SAME_HOST(0),
+ SAME_SUBNET(1),
+ SAME_SITE(32),
+ SAME_REGION(64),
+ SAME_CONTINENT(128),
+ UNRESTRICTED(255);
+
+ private final int ttl;
+
+ MulticastTimeToLive(final int ttl) {
+ this.ttl = ttl;
+ }
+
+ public int getTtl() {
+ return ttl;
+ }
+
+ public MulticastTimeToLive valueOfByTtl(final int ttl) {
+ for (final MulticastTimeToLive value : values()) {
+ if (value.getTtl() == ttl) {
+ return value;
+ }
+ }
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/MulticastUtils.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/MulticastUtils.java b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/MulticastUtils.java
new file mode 100644
index 0000000..84d76d2
--- /dev/null
+++ b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/MulticastUtils.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.io.socket.multicast;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.MulticastSocket;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ */
+public final class MulticastUtils {
+
+ private static final Logger logger = new org.apache.nifi.logging.NiFiLog(LoggerFactory.getLogger(MulticastUtils.class));
+
+ public static MulticastSocket createMulticastSocket(final MulticastConfiguration config) throws IOException {
+ return createMulticastSocket(0, config);
+ }
+
+ public static MulticastSocket createMulticastSocket(final int port, final MulticastConfiguration config) throws IOException {
+ if (config == null) {
+ throw new IllegalArgumentException("Configuration may not be null.");
+ }
+
+ final MulticastSocket socket;
+ if (port <= 0) {
+ socket = new MulticastSocket();
+ } else {
+ socket = new MulticastSocket(port);
+ }
+ socket.setTimeToLive(config.getTtl().getTtl());
+
+ if (config.getSocketTimeout() != null) {
+ socket.setSoTimeout(config.getSocketTimeout());
+ }
+
+ if (config.getReuseAddress() != null) {
+ socket.setReuseAddress(config.getReuseAddress());
+ }
+
+ if (config.getReceiveBufferSize() != null) {
+ socket.setReceiveBufferSize(config.getReceiveBufferSize());
+ }
+
+ if (config.getSendBufferSize() != null) {
+ socket.setSendBufferSize(config.getSendBufferSize());
+ }
+
+ if (config.getTrafficClass() != null) {
+ socket.setTrafficClass(config.getTrafficClass());
+ }
+
+ if (config.getLoopbackMode() != null) {
+ socket.setLoopbackMode(config.getLoopbackMode());
+ }
+
+ return socket;
+ }
+
+ public static void closeQuietly(final MulticastSocket socket) {
+
+ if (socket == null) {
+ return;
+ }
+
+ try {
+ socket.close();
+ } catch (final Exception ex) {
+ logger.debug("Failed to close multicast socket due to: " + ex, ex);
+ }
+
+ }
+
+ public static void closeQuietly(final MulticastSocket socket, final InetAddress groupAddress) {
+
+ if (socket == null) {
+ return;
+ }
+
+ try {
+ socket.leaveGroup(groupAddress);
+ } catch (final Exception ex) {
+ logger.debug("Failed to leave multicast group due to: " + ex, ex);
+ }
+
+ try {
+ socket.close();
+ } catch (final Exception ex) {
+ logger.debug("Failed to close multicast socket due to: " + ex, ex);
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/ServiceDiscovery.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/ServiceDiscovery.java b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/ServiceDiscovery.java
new file mode 100644
index 0000000..74c1a30
--- /dev/null
+++ b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/ServiceDiscovery.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.io.socket.multicast;
+
+/**
+ * Defines a generic interface for discovering services.
+ *
+ */
+public interface ServiceDiscovery {
+
+ /**
+ * @return the discovered service
+ */
+ DiscoverableService getService();
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/ServicesBroadcaster.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/ServicesBroadcaster.java b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/ServicesBroadcaster.java
new file mode 100644
index 0000000..2a9f9b2
--- /dev/null
+++ b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/multicast/ServicesBroadcaster.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.io.socket.multicast;
+
+import java.util.Set;
+
+/**
+ * Defines the interface for broadcasting a collection of services for client
+ * discovery.
+ *
+ */
+public interface ServicesBroadcaster {
+
+ /**
+ * @return the delay in milliseconds to wait between successive broadcasts
+ */
+ int getBroadcastDelayMs();
+
+ /**
+ * @return the broadcasted services
+ */
+ Set<DiscoverableService> getServices();
+
+ /**
+ * Adds the given service to the set of broadcasted services.
+ *
+ * @param service a service
+ * @return true if the service was added to the set; false a service with
+ * the given service name already exists in the set.
+ */
+ boolean addService(DiscoverableService service);
+
+ /**
+ * Removes the service with the given service name from the set.
+ *
+ * @param serviceName a service name
+ * @return true if the service was removed; false otherwise
+ */
+ boolean removeService(String serviceName);
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-socket-utils/src/test/java/org/apache/nifi/io/nio/example/ServerMain.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-socket-utils/src/test/java/org/apache/nifi/io/nio/example/ServerMain.java b/nifi-commons/nifi-socket-utils/src/test/java/org/apache/nifi/io/nio/example/ServerMain.java
new file mode 100644
index 0000000..a266ade
--- /dev/null
+++ b/nifi-commons/nifi-socket-utils/src/test/java/org/apache/nifi/io/nio/example/ServerMain.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.io.nio.example;
+
+import java.io.IOException;
+import java.util.Calendar;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.nifi.io.nio.BufferPool;
+import org.apache.nifi.io.nio.ChannelListener;
+import org.apache.nifi.io.nio.consumer.StreamConsumer;
+import org.apache.nifi.io.nio.consumer.StreamConsumerFactory;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ */
+public final class ServerMain {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(ServerMain.class);
+
+ public static void main(final String[] args) throws IOException {
+ System.setProperty("org.slf4j.simpleLogger.defaultLogLevel", "info");
+ System.setProperty("org.slf4j.simpleLogger.log.nifi.io.nio", "debug");
+
+ final ScheduledExecutorService executor = Executors.newScheduledThreadPool(10);
+ final Map<StreamConsumer, ScheduledFuture<?>> consumerMap = new ConcurrentHashMap<>();
+ final BufferPool bufferPool = new BufferPool(10, 5 << 20, false, 40.0);
+ ChannelListener listener = null;
+ try {
+ executor.scheduleWithFixedDelay(bufferPool, 0L, 5L, TimeUnit.SECONDS);
+ listener = new ChannelListener(5, new ExampleStreamConsumerFactory(executor, consumerMap), bufferPool, 5, TimeUnit.MILLISECONDS, false);
+ listener.setChannelReaderSchedulingPeriod(50L, TimeUnit.MILLISECONDS);
+ listener.addDatagramChannel(null, 20000, 32 << 20);
+ LOGGER.info("Listening for UDP data on port 20000");
+ listener.addServerSocket(null, 20001, 64 << 20);
+ LOGGER.info("listening for TCP connections on port 20001");
+ listener.addServerSocket(null, 20002, 64 << 20);
+ LOGGER.info("listening for TCP connections on port 20002");
+ final Calendar endTime = Calendar.getInstance();
+ endTime.add(Calendar.MINUTE, 30);
+ while (true) {
+ processAllConsumers(consumerMap);
+ if (endTime.before(Calendar.getInstance())) {
+ break; // time to shut down
+ }
+ }
+ } finally {
+ if (listener != null) {
+ LOGGER.info("Shutting down server....");
+ listener.shutdown(1L, TimeUnit.SECONDS);
+ LOGGER.info("Consumer map size = " + consumerMap.size());
+ while (consumerMap.size() > 0) {
+ processAllConsumers(consumerMap);
+ }
+ LOGGER.info("Consumer map size = " + consumerMap.size());
+ }
+ executor.shutdown();
+ }
+ }
+
+ private static void processAllConsumers(final Map<StreamConsumer, ScheduledFuture<?>> consumerMap) {
+ final Set<StreamConsumer> deadConsumers = new HashSet<>();
+ for (final Map.Entry<StreamConsumer, ScheduledFuture<?>> entry : consumerMap.entrySet()) {
+ if (entry.getKey().isConsumerFinished()) {
+ entry.getValue().cancel(true);
+ deadConsumers.add(entry.getKey());
+ }
+ }
+ for (final StreamConsumer consumer : deadConsumers) {
+ LOGGER.debug("removing consumer " + consumer);
+ consumerMap.remove(consumer);
+ }
+ }
+
+ public static final class ConsumerRunner implements Runnable {
+
+ private final StreamConsumer consumer;
+
+ public ConsumerRunner(final StreamConsumer consumer) {
+ this.consumer = consumer;
+ }
+
+ @Override
+ public void run() {
+ if (consumer.isConsumerFinished()) {
+ return;
+ }
+ try {
+ consumer.process();
+ } catch (IOException ex) {
+ LOGGER.error("", ex);
+ }
+ }
+ }
+
+ public static final class ExampleStreamConsumerFactory implements StreamConsumerFactory {
+
+ final ScheduledExecutorService executor;
+ final Map<StreamConsumer, ScheduledFuture<?>> consumerMap;
+
+ public ExampleStreamConsumerFactory(final ScheduledExecutorService executor, final Map<StreamConsumer, ScheduledFuture<?>> consumerMap) {
+ this.executor = executor;
+ this.consumerMap = consumerMap;
+ }
+
+ @Override
+ public StreamConsumer newInstance(final String streamId) {
+ final StreamConsumer consumer = new UselessStreamConsumer(streamId);
+ final ScheduledFuture<?> future = executor.scheduleWithFixedDelay(new ConsumerRunner(consumer), 0L, 10L, TimeUnit.MILLISECONDS);
+ consumerMap.put(consumer, future);
+ LOGGER.info("Added consumer: " + consumer);
+ return consumer;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-socket-utils/src/test/java/org/apache/nifi/io/nio/example/TCPClient.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-socket-utils/src/test/java/org/apache/nifi/io/nio/example/TCPClient.java b/nifi-commons/nifi-socket-utils/src/test/java/org/apache/nifi/io/nio/example/TCPClient.java
new file mode 100644
index 0000000..1c4b70c
--- /dev/null
+++ b/nifi-commons/nifi-socket-utils/src/test/java/org/apache/nifi/io/nio/example/TCPClient.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.io.nio.example;
+
+import java.io.IOException;
+import java.net.Socket;
+import java.net.SocketException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ */
+public class TCPClient {
+
+ private static final Logger logger = LoggerFactory.getLogger(TCPClient.class);
+
+ public static void main(final String[] args) throws Exception {
+ final byte[] bytes = TCPClient.makeBytes();
+ Thread first = new Thread(new Runnable() {
+
+ @Override
+ public void run() {
+ try {
+ for (int i = 0; i < 10; i++) {
+ sendData(20001, bytes);
+ }
+ } catch (Exception e) {
+ logger.error("Blew exception", e);
+ }
+ }
+ });
+ Thread second = new Thread(new Runnable() {
+
+ @Override
+ public void run() {
+ try {
+ for (int i = 0; i < 10; i++) {
+ sendData(20002, bytes);
+ }
+ } catch (Exception e) {
+ logger.error("Blew exception", e);
+ }
+ }
+ });
+ first.start();
+ second.start();
+ }
+
+ public static byte[] makeBytes() {
+ byte[] bytes = new byte[2 << 20];
+ return bytes;
+ }
+
+ private static void sendData(final int port, final byte[] bytes) throws SocketException, IOException, InterruptedException {
+ long totalBytes;
+ try (Socket sock = new Socket("localhost", port)) {
+ sock.setTcpNoDelay(true);
+ sock.setSoTimeout(2000);
+ totalBytes = 0L;
+ logger.info("socket established " + sock + " to port " + port + " now waiting 5 seconds to send anything...");
+ Thread.sleep(5000L);
+ for (int i = 0; i < 1000; i++) {
+ sock.getOutputStream().write(bytes);
+ totalBytes += bytes.length;
+ }
+ sock.getOutputStream().flush();
+ }
+ logger.info("Total bytes sent: " + totalBytes + " to port " + port);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-socket-utils/src/test/java/org/apache/nifi/io/nio/example/UDPClient.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-socket-utils/src/test/java/org/apache/nifi/io/nio/example/UDPClient.java b/nifi-commons/nifi-socket-utils/src/test/java/org/apache/nifi/io/nio/example/UDPClient.java
new file mode 100644
index 0000000..00a00a1
--- /dev/null
+++ b/nifi-commons/nifi-socket-utils/src/test/java/org/apache/nifi/io/nio/example/UDPClient.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.io.nio.example;
+
+import java.net.DatagramPacket;
+import java.net.DatagramSocket;
+import java.net.InetSocketAddress;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ */
+public class UDPClient {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(UDPClient.class);
+
+ public static void main(final String[] args) throws Exception {
+ final byte[] buffer = UDPClient.makeBytes();
+ final DatagramPacket packet = new DatagramPacket(buffer, buffer.length, new InetSocketAddress("localhost", 20000));
+ final DatagramSocket socket = new DatagramSocket();
+ final long startTime = System.nanoTime();
+ for (int i = 0; i < 819200; i++) { // 100 MB
+ socket.send(packet);
+ }
+ final long endTime = System.nanoTime();
+ final long durationMillis = (endTime - startTime) / 1000000;
+ LOGGER.info("Sent all UDP packets without any obvious errors | duration ms= " + durationMillis);
+ }
+
+ public static byte[] makeBytes() {
+ byte[] bytes = new byte[128];
+ return bytes;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-socket-utils/src/test/java/org/apache/nifi/io/nio/example/UselessStreamConsumer.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-socket-utils/src/test/java/org/apache/nifi/io/nio/example/UselessStreamConsumer.java b/nifi-commons/nifi-socket-utils/src/test/java/org/apache/nifi/io/nio/example/UselessStreamConsumer.java
new file mode 100644
index 0000000..107c087
--- /dev/null
+++ b/nifi-commons/nifi-socket-utils/src/test/java/org/apache/nifi/io/nio/example/UselessStreamConsumer.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.io.nio.example;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.nifi.io.nio.consumer.AbstractStreamConsumer;
+
+/**
+ *
+ */
+public class UselessStreamConsumer extends AbstractStreamConsumer {
+
+ public UselessStreamConsumer(final String id) {
+ super(id);
+ }
+
+ @Override
+ protected void processBuffer(final ByteBuffer buffer) throws IOException {
+ }
+
+ @Override
+ protected void onConsumerDone() {
+ System.err.println("IN consumer done");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-socket-utils/src/test/resources/log4j.xml
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-socket-utils/src/test/resources/log4j.xml b/nifi-commons/nifi-socket-utils/src/test/resources/log4j.xml
new file mode 100644
index 0000000..8e93769
--- /dev/null
+++ b/nifi-commons/nifi-socket-utils/src/test/resources/log4j.xml
@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+ <!-- Appender for printing formatted log statements to the console. -->
+ <appender name="console" class="org.apache.log4j.ConsoleAppender">
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %-5p [%t] %40.40c - %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- Logger for managing logging statements for nifi -->
+ <logger name="nifi">
+ <level value="debug"/>
+ </logger>
+
+ <root>
+ <level value="warn"/>
+ <appender-ref ref="console"/>
+ </root>
+</log4j:configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/.gitignore
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/.gitignore b/nifi-commons/nifi-utils/.gitignore
new file mode 100755
index 0000000..12c5231
--- /dev/null
+++ b/nifi-commons/nifi-utils/.gitignore
@@ -0,0 +1,8 @@
+/target
+/target
+/target
+/target
+/target
+/target
+/target
+/target
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/pom.xml b/nifi-commons/nifi-utils/pom.xml
new file mode 100644
index 0000000..6927fc0
--- /dev/null
+++ b/nifi-commons/nifi-utils/pom.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-commons</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>nifi-utils</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ <packaging>jar</packaging>
+ <!--
+ This project intentionally has no additional dependencies beyond that pulled in by the parent. It is a general purpose utility library
+ and should keep its surface/tension minimal.
+ -->
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/flowfile/attributes/CoreAttributes.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/flowfile/attributes/CoreAttributes.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/flowfile/attributes/CoreAttributes.java
new file mode 100644
index 0000000..9b4c3af
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/flowfile/attributes/CoreAttributes.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.flowfile.attributes;
+
+public enum CoreAttributes implements FlowFileAttributeKey {
+
+ /**
+ * The flowfile's path indicates the relative directory to which a FlowFile belongs and does not contain the filename
+ */
+ PATH("path"),
+ /**
+ * The flowfile's absolute path indicates the absolute directory to which a FlowFile belongs and does not contain the filename
+ */
+ ABSOLUTE_PATH("absolute.path"),
+ /**
+ * The filename of the FlowFile. The filename should not contain any directory structure.
+ */
+ FILENAME("filename"),
+ /**
+ * A unique UUID assigned to this FlowFile
+ */
+ UUID("uuid"),
+ /**
+ * A numeric value indicating the FlowFile priority
+ */
+ PRIORITY("priority"),
+ /**
+ * The MIME Type of this FlowFile
+ */
+ MIME_TYPE("mime.type"),
+ /**
+ * Specifies the reason that a FlowFile is being discarded
+ */
+ DISCARD_REASON("discard.reason"),
+ /**
+ * Indicates an identifier other than the FlowFile's UUID that is known to refer to this FlowFile.
+ */
+ ALTERNATE_IDENTIFIER("alternate.identifier");
+
+ private final String key;
+
+ private CoreAttributes(final String key) {
+ this.key = key;
+ }
+
+ @Override
+ public String key() {
+ return key;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/flowfile/attributes/FlowFileAttributeKey.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/flowfile/attributes/FlowFileAttributeKey.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/flowfile/attributes/FlowFileAttributeKey.java
new file mode 100644
index 0000000..9637631
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/flowfile/attributes/FlowFileAttributeKey.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.flowfile.attributes;
+
+public interface FlowFileAttributeKey {
+
+ String key();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/StandardVersionNegotiator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/StandardVersionNegotiator.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/StandardVersionNegotiator.java
new file mode 100644
index 0000000..77c34c9
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/StandardVersionNegotiator.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+public class StandardVersionNegotiator implements VersionNegotiator {
+
+ private final List<Integer> versions;
+ private int curVersion;
+
+ public StandardVersionNegotiator(final int... supportedVersions) {
+ if (Objects.requireNonNull(supportedVersions).length == 0) {
+ throw new IllegalArgumentException("At least one version must be supported");
+ }
+
+ final List<Integer> supported = new ArrayList<>();
+ for (final int version : supportedVersions) {
+ supported.add(version);
+ }
+ this.versions = Collections.unmodifiableList(supported);
+ this.curVersion = supportedVersions[0];
+ }
+
+ @Override
+ public int getVersion() {
+ return curVersion;
+ }
+
+ @Override
+ public void setVersion(final int version) throws IllegalArgumentException {
+ if (!isVersionSupported(version)) {
+ throw new IllegalArgumentException("Version " + version + " is not supported");
+ }
+
+ this.curVersion = version;
+ }
+
+ @Override
+ public int getPreferredVersion() {
+ return versions.get(0);
+ }
+
+ @Override
+ public Integer getPreferredVersion(final int maxVersion) {
+ for (final Integer version : this.versions) {
+ if (maxVersion >= version) {
+ return version;
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public boolean isVersionSupported(final int version) {
+ return versions.contains(version);
+ }
+
+ @Override
+ public List<Integer> getSupportedVersions() {
+ return versions;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/VersionNegotiator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/VersionNegotiator.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/VersionNegotiator.java
new file mode 100644
index 0000000..d8ee27a
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/VersionNegotiator.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote;
+
+import java.util.List;
+
+public interface VersionNegotiator {
+
+ /**
+ * @return the currently configured Version of this resource
+ */
+ int getVersion();
+
+ /**
+ * Sets the version of this resource to the specified version. Only the lower byte of the version is relevant.
+ *
+ * @param version the version to set
+ * @throws IllegalArgumentException if the given Version is not supported by this resource, as is indicated by the {@link #isVersionSupported(int)} method
+ */
+ void setVersion(int version) throws IllegalArgumentException;
+
+ /**
+ *
+ * @return the Version of this resource that is preferred
+ */
+ int getPreferredVersion();
+
+ /**
+ * Gets the preferred version of this resource that is no greater than the given maxVersion. If no acceptable version exists that is less than <code>maxVersion</code>, then <code>null</code> is
+ * returned
+ *
+ * @param maxVersion the maximum version desired
+ * @return the preferred version if found; null otherwise
+ */
+ Integer getPreferredVersion(int maxVersion);
+
+ /**
+ * Indicates whether or not the specified version is supported by this resource
+ *
+ * @param version the version to test
+ * @return true if supported; false otherwise
+ */
+ boolean isVersionSupported(int version);
+
+ List<Integer> getSupportedVersions();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/exception/TransmissionDisabledException.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/exception/TransmissionDisabledException.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/exception/TransmissionDisabledException.java
new file mode 100644
index 0000000..d18c807
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/exception/TransmissionDisabledException.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.exception;
+
+/**
+ * Indicates that the user disabled transmission while communications were taking place with a peer
+ */
+public class TransmissionDisabledException extends RuntimeException {
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/CompressionInputStream.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/CompressionInputStream.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/CompressionInputStream.java
new file mode 100644
index 0000000..6434b2d
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/CompressionInputStream.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.io;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
+import java.util.zip.DataFormatException;
+import java.util.zip.Inflater;
+
+public class CompressionInputStream extends InputStream {
+
+ private final InputStream in;
+ private final Inflater inflater;
+
+ private byte[] compressedBuffer;
+ private byte[] buffer;
+
+ private int bufferIndex;
+ private boolean eos = false; // whether or not we've reached the end of stream
+ private boolean allDataRead = false; // different from eos b/c eos means allDataRead == true && buffer is empty
+
+ private final byte[] fourByteBuffer = new byte[4];
+
+ public CompressionInputStream(final InputStream in) {
+ this.in = in;
+ inflater = new Inflater();
+
+ buffer = new byte[0];
+ compressedBuffer = new byte[0];
+ bufferIndex = 1;
+ }
+
+ private String toHex(final byte[] array) {
+ final StringBuilder sb = new StringBuilder("0x");
+ for (final byte b : array) {
+ final String hex = Integer.toHexString(b).toUpperCase();
+ if (hex.length() == 1) {
+ sb.append("0");
+ }
+ sb.append(hex);
+ }
+ return sb.toString();
+ }
+
+ protected void readChunkHeader() throws IOException {
+ // Ensure that we have a valid SYNC chunk
+ fillBuffer(fourByteBuffer);
+ if (!Arrays.equals(CompressionOutputStream.SYNC_BYTES, fourByteBuffer)) {
+ throw new IOException("Invalid CompressionInputStream. Expected first 4 bytes to be 'SYNC' but were " + toHex(fourByteBuffer));
+ }
+
+ // determine the size of the decompressed buffer
+ fillBuffer(fourByteBuffer);
+ buffer = new byte[toInt(fourByteBuffer)];
+
+ // determine the size of the compressed buffer
+ fillBuffer(fourByteBuffer);
+ compressedBuffer = new byte[toInt(fourByteBuffer)];
+
+ bufferIndex = buffer.length; // indicate that buffer is empty
+ }
+
+ private int toInt(final byte[] data) {
+ return ((data[0] & 0xFF) << 24)
+ | ((data[1] & 0xFF) << 16)
+ | ((data[2] & 0xFF) << 8)
+ | (data[3] & 0xFF);
+ }
+
+ protected void bufferAndDecompress() throws IOException {
+ if (allDataRead) {
+ eos = true;
+ return;
+ }
+
+ readChunkHeader();
+ fillBuffer(compressedBuffer);
+
+ inflater.setInput(compressedBuffer);
+ try {
+ inflater.inflate(buffer);
+ } catch (final DataFormatException e) {
+ throw new IOException(e);
+ }
+ inflater.reset();
+
+ bufferIndex = 0;
+ final int moreDataByte = in.read();
+ if (moreDataByte < 1) {
+ allDataRead = true;
+ } else if (moreDataByte > 1) {
+ throw new IOException("Expected indicator of whether or not more data was to come (-1, 0, or 1) but got " + moreDataByte);
+ }
+ }
+
+ private void fillBuffer(final byte[] buffer) throws IOException {
+ int len;
+ int bytesLeft = buffer.length;
+ int bytesRead = 0;
+ while (bytesLeft > 0 && (len = in.read(buffer, bytesRead, bytesLeft)) > 0) {
+ bytesLeft -= len;
+ bytesRead += len;
+ }
+
+ if (bytesRead < buffer.length) {
+ throw new EOFException();
+ }
+ }
+
+ private boolean isBufferEmpty() {
+ return bufferIndex >= buffer.length;
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (eos) {
+ return -1;
+ }
+
+ if (isBufferEmpty()) {
+ bufferAndDecompress();
+ }
+
+ if (isBufferEmpty()) {
+ eos = true;
+ return -1;
+ }
+
+ return buffer[bufferIndex++] & 0xFF;
+ }
+
+ @Override
+ public int read(final byte[] b) throws IOException {
+ return read(b, 0, b.length);
+ }
+
+ @Override
+ public int read(final byte[] b, final int off, final int len) throws IOException {
+ if (eos) {
+ return -1;
+ }
+
+ if (isBufferEmpty()) {
+ bufferAndDecompress();
+ }
+
+ if (isBufferEmpty()) {
+ eos = true;
+ return -1;
+ }
+
+ final int free = buffer.length - bufferIndex;
+ final int bytesToTransfer = Math.min(len, free);
+ System.arraycopy(buffer, bufferIndex, b, off, bytesToTransfer);
+ bufferIndex += bytesToTransfer;
+
+ return bytesToTransfer;
+ }
+
+ /**
+ * Does nothing. Does NOT close underlying InputStream
+ *
+ * @throws java.io.IOException for any issues closing underlying stream
+ */
+ @Override
+ public void close() throws IOException {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/CompressionOutputStream.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/CompressionOutputStream.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/CompressionOutputStream.java
new file mode 100644
index 0000000..525b5b1
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/CompressionOutputStream.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.io;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.zip.Deflater;
+
+public class CompressionOutputStream extends OutputStream {
+
+ public static final byte[] SYNC_BYTES = new byte[]{'S', 'Y', 'N', 'C'};
+
+ public static final int DEFAULT_COMPRESSION_LEVEL = 1;
+ public static final int DEFAULT_BUFFER_SIZE = 64 << 10;
+ public static final int MIN_BUFFER_SIZE = 8 << 10;
+
+ private final OutputStream out;
+ private final Deflater deflater;
+
+ private final byte[] buffer;
+ private final byte[] compressed;
+
+ private int bufferIndex = 0;
+ private boolean dataWritten = false;
+
+ public CompressionOutputStream(final OutputStream outStream) {
+ this(outStream, DEFAULT_BUFFER_SIZE);
+ }
+
+ public CompressionOutputStream(final OutputStream outStream, final int bufferSize) {
+ this(outStream, bufferSize, DEFAULT_COMPRESSION_LEVEL, Deflater.DEFAULT_STRATEGY);
+ }
+
+ public CompressionOutputStream(final OutputStream outStream, final int bufferSize, final int level, final int strategy) {
+ if (bufferSize < MIN_BUFFER_SIZE) {
+ throw new IllegalArgumentException("Buffer size must be at least " + MIN_BUFFER_SIZE);
+ }
+
+ this.out = outStream;
+ this.deflater = new Deflater(level);
+ this.deflater.setStrategy(strategy);
+ buffer = new byte[bufferSize];
+ compressed = new byte[bufferSize + 64];
+ }
+
+ /**
+ * Compresses the currently buffered chunk of data and sends it to the output stream
+ *
+ * @throws IOException if issues occur writing to stream
+ */
+ protected void compressAndWrite() throws IOException {
+ if (bufferIndex <= 0) {
+ return;
+ }
+
+ deflater.setInput(buffer, 0, bufferIndex);
+ deflater.finish();
+ final int compressedBytes = deflater.deflate(compressed);
+
+ writeChunkHeader(compressedBytes);
+ out.write(compressed, 0, compressedBytes);
+
+ bufferIndex = 0;
+ deflater.reset();
+ }
+
+ private void writeChunkHeader(final int compressedBytes) throws IOException {
+ // If we have already written data, write out a '1' to indicate that we have more data; when we close
+ // the stream, we instead write a '0' to indicate that we are finished sending data.
+ if (dataWritten) {
+ out.write(1);
+ }
+ out.write(SYNC_BYTES);
+ dataWritten = true;
+
+ writeInt(out, bufferIndex);
+ writeInt(out, compressedBytes);
+ }
+
+ private void writeInt(final OutputStream out, final int val) throws IOException {
+ out.write(val >>> 24);
+ out.write(val >>> 16);
+ out.write(val >>> 8);
+ out.write(val);
+ }
+
+ protected boolean bufferFull() {
+ return bufferIndex >= buffer.length;
+ }
+
+ @Override
+ public void write(final int b) throws IOException {
+ buffer[bufferIndex++] = (byte) (b & 0xFF);
+ if (bufferFull()) {
+ compressAndWrite();
+ }
+ }
+
+ @Override
+ public void write(final byte[] b) throws IOException {
+ write(b, 0, b.length);
+ }
+
+ @Override
+ public void write(final byte[] b, final int off, final int len) throws IOException {
+ int bytesLeft = len;
+ while (bytesLeft > 0) {
+ final int free = buffer.length - bufferIndex;
+ final int bytesThisIteration = Math.min(bytesLeft, free);
+ System.arraycopy(b, off + len - bytesLeft, buffer, bufferIndex, bytesThisIteration);
+ bufferIndex += bytesThisIteration;
+
+ bytesLeft -= bytesThisIteration;
+ if (bufferFull()) {
+ compressAndWrite();
+ }
+ }
+ }
+
+ @Override
+ public void flush() throws IOException {
+ compressAndWrite();
+ super.flush();
+ }
+
+ @Override
+ public void close() throws IOException {
+ compressAndWrite();
+ out.write(0); // indicate that the stream is finished.
+ out.flush();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/InterruptableInputStream.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/InterruptableInputStream.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/InterruptableInputStream.java
new file mode 100644
index 0000000..e03dfbf
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/InterruptableInputStream.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.io;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.nifi.remote.exception.TransmissionDisabledException;
+
+public class InterruptableInputStream extends InputStream {
+
+ private volatile boolean interrupted = false;
+ private final InputStream in;
+
+ public InterruptableInputStream(final InputStream in) {
+ this.in = in;
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (interrupted) {
+ throw new TransmissionDisabledException();
+ }
+
+ return in.read();
+ }
+
+ @Override
+ public int read(byte[] b) throws IOException {
+ if (interrupted) {
+ throw new TransmissionDisabledException();
+ }
+
+ return in.read(b);
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ if (interrupted) {
+ throw new TransmissionDisabledException();
+ }
+
+ return in.read(b, off, len);
+ }
+
+ @Override
+ public int available() throws IOException {
+ if (interrupted) {
+ throw new TransmissionDisabledException();
+ }
+
+ return in.available();
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (interrupted) {
+ throw new TransmissionDisabledException();
+ }
+
+ in.close();
+ }
+
+ @Override
+ public synchronized void mark(int readlimit) {
+ if (interrupted) {
+ throw new TransmissionDisabledException();
+ }
+
+ in.mark(readlimit);
+ }
+
+ @Override
+ public boolean markSupported() {
+ if (interrupted) {
+ throw new TransmissionDisabledException();
+ }
+
+ return in.markSupported();
+ }
+
+ @Override
+ public synchronized void reset() throws IOException {
+ if (interrupted) {
+ throw new TransmissionDisabledException();
+ }
+
+ in.reset();
+ }
+
+ @Override
+ public long skip(long n) throws IOException {
+ if (interrupted) {
+ throw new TransmissionDisabledException();
+ }
+
+ return in.skip(n);
+ }
+
+ public void interrupt() {
+ interrupted = true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/InterruptableOutputStream.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/InterruptableOutputStream.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/InterruptableOutputStream.java
new file mode 100644
index 0000000..cba5be6
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/InterruptableOutputStream.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.io;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+import org.apache.nifi.remote.exception.TransmissionDisabledException;
+
+public class InterruptableOutputStream extends OutputStream {
+
+ private final OutputStream out;
+ private volatile boolean interrupted = false;
+
+ public InterruptableOutputStream(final OutputStream out) {
+ this.out = out;
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ if (interrupted) {
+ throw new TransmissionDisabledException();
+ }
+
+ out.write(b);
+ }
+
+ @Override
+ public void write(byte[] b) throws IOException {
+ if (interrupted) {
+ throw new TransmissionDisabledException();
+ }
+
+ out.write(b);
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) throws IOException {
+ if (interrupted) {
+ throw new TransmissionDisabledException();
+ }
+
+ out.write(b, off, len);
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (interrupted) {
+ throw new TransmissionDisabledException();
+ }
+
+ out.close();
+ }
+
+ @Override
+ public void flush() throws IOException {
+ if (interrupted) {
+ throw new TransmissionDisabledException();
+ }
+
+ out.flush();
+ }
+
+ public void interrupt() {
+ this.interrupted = true;
+ }
+}
[12/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__/pom.xml b/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__/pom.xml
new file mode 100644
index 0000000..fdbb58b
--- /dev/null
+++ b/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__/pom.xml
@@ -0,0 +1,59 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>${groupId}</groupId>
+ <artifactId>${rootArtifactId}</artifactId>
+ <version>${version}</version>
+ </parent>
+
+ <artifactId>nifi-${artifactBaseName}</artifactId>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>${groupId}</groupId>
+ <artifactId>nifi-${artifactBaseName}-api</artifactId>
+ <version>${version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-processor-utils</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-mock</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-simple</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__/src/main/java/StandardMyService.java
----------------------------------------------------------------------
diff --git a/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__/src/main/java/StandardMyService.java b/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__/src/main/java/StandardMyService.java
new file mode 100644
index 0000000..193e6b7
--- /dev/null
+++ b/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__/src/main/java/StandardMyService.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ${package};
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnDisabled;
+import org.apache.nifi.annotation.lifecycle.OnEnabled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.controller.AbstractControllerService;
+import org.apache.nifi.controller.ConfigurationContext;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.reporting.InitializationException;
+
+@Tags({ "example"})
+@CapabilityDescription("Example ControllerService implementation of MyService.")
+public class StandardMyService extends AbstractControllerService implements MyService {
+
+ public static final PropertyDescriptor MY_PROPERTY = new PropertyDescriptor
+ .Builder().name("My Property")
+ .description("Example Property")
+ .required(true)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+
+ private static final List<PropertyDescriptor> properties;
+
+ static {
+ final List<PropertyDescriptor> props = new ArrayList<>();
+ props.add(MY_PROPERTY);
+ properties = Collections.unmodifiableList(props);
+ }
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return properties;
+ }
+
+ /**
+ * @param context
+ * the configuration context
+ * @throws InitializationException
+ * if unable to create a database connection
+ */
+ @OnEnabled
+ public void onEnabled(final ConfigurationContext context) throws InitializationException {
+
+ }
+
+ @OnDisabled
+ public void shutdown() {
+
+ }
+
+ @Override
+ public void execute() throws ProcessException {
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService
----------------------------------------------------------------------
diff --git a/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService b/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService
new file mode 100644
index 0000000..811fcd3
--- /dev/null
+++ b/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService
@@ -0,0 +1,15 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+${package}.StandardMyService
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__/src/test/java/TestProcessor.java
----------------------------------------------------------------------
diff --git a/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__/src/test/java/TestProcessor.java b/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__/src/test/java/TestProcessor.java
new file mode 100644
index 0000000..be00a70
--- /dev/null
+++ b/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__/src/test/java/TestProcessor.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ${package};
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.exception.ProcessException;
+
+public class TestProcessor extends AbstractProcessor {
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ }
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ List<PropertyDescriptor> propDescs = new ArrayList<>();
+ propDescs.add(new PropertyDescriptor.Builder()
+ .name("MyService test processor")
+ .description("MyService test processor")
+ .identifiesControllerService(MyService.class)
+ .required(true)
+ .build());
+ return propDescs;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__/src/test/java/TestStandardMyService.java
----------------------------------------------------------------------
diff --git a/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__/src/test/java/TestStandardMyService.java b/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__/src/test/java/TestStandardMyService.java
new file mode 100644
index 0000000..3f430e6
--- /dev/null
+++ b/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__/src/test/java/TestStandardMyService.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ${package};
+
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestStandardMyService {
+
+ @Before
+ public void init() {
+
+ }
+
+ @Test
+ public void testService() throws InitializationException {
+ final TestRunner runner = TestRunners.newTestRunner(TestProcessor.class);
+ final StandardMyService service = new StandardMyService();
+ runner.addControllerService("test-good", service);
+
+ runner.setProperty(service, StandardMyService.MY_PROPERTY, "test-value");
+ runner.enableControllerService(service);
+
+ runner.assertValid(service);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/pom.xml b/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/pom.xml
new file mode 100644
index 0000000..0f32431
--- /dev/null
+++ b/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/pom.xml
@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-nar-bundles</artifactId>
+ <version>${nifiVersion}</version>
+ </parent>
+
+ <groupId>${groupId}</groupId>
+ <artifactId>${artifactId}</artifactId>
+ <version>${version}</version>
+ <packaging>pom</packaging>
+
+ <modules>
+ <module>nifi-${artifactBaseName}-api</module>
+ <module>nifi-${artifactBaseName}-api-nar</module>
+ <module>nifi-${artifactBaseName}</module>
+ <module>nifi-${artifactBaseName}-nar</module>
+ </modules>
+
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-maven-archetypes/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-maven-archetypes/pom.xml b/nifi-maven-archetypes/pom.xml
new file mode 100644
index 0000000..b817604
--- /dev/null
+++ b/nifi-maven-archetypes/pom.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-maven-archetypes</artifactId>
+ <packaging>pom</packaging>
+ <modules>
+ <module>nifi-processor-bundle-archetype</module>
+ <module>nifi-service-bundle-archetype</module>
+ </modules>
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-mock/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-mock/pom.xml b/nifi-mock/pom.xml
new file mode 100644
index 0000000..8db3830
--- /dev/null
+++ b/nifi-mock/pom.xml
@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>nifi-mock</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-utils</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-expression-language</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-data-provenance-utils</artifactId>
+ </dependency>
+ <dependency>
+ <!-- Dependency marked as provided, not test, because we have assertion
+ methods in our MockSession & MockFlowFile -->
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-mock/src/main/java/org/apache/nifi/provenance/MockProvenanceEventRepository.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/main/java/org/apache/nifi/provenance/MockProvenanceEventRepository.java b/nifi-mock/src/main/java/org/apache/nifi/provenance/MockProvenanceEventRepository.java
new file mode 100644
index 0000000..241041a
--- /dev/null
+++ b/nifi-mock/src/main/java/org/apache/nifi/provenance/MockProvenanceEventRepository.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.nifi.events.EventReporter;
+import org.apache.nifi.provenance.lineage.ComputeLineageSubmission;
+import org.apache.nifi.provenance.search.Query;
+import org.apache.nifi.provenance.search.QuerySubmission;
+import org.apache.nifi.provenance.search.SearchableField;
+
+public class MockProvenanceEventRepository implements ProvenanceEventRepository {
+
+ private final List<ProvenanceEventRecord> records = new ArrayList<>();
+ private final AtomicLong idGenerator = new AtomicLong(0L);
+
+ @Override
+ public void registerEvents(final Iterable<ProvenanceEventRecord> events) {
+ for (final ProvenanceEventRecord event : events) {
+ registerEvent(event);
+ }
+ }
+
+ @Override
+ public void registerEvent(final ProvenanceEventRecord event) {
+ final StandardProvenanceEventRecord newRecord;
+ if (event instanceof StandardProvenanceEventRecord) {
+ newRecord = (StandardProvenanceEventRecord) event;
+ } else {
+ newRecord = new StandardProvenanceEventRecord.Builder().fromEvent(event).build();
+ }
+ newRecord.setEventId(idGenerator.getAndIncrement());
+
+ records.add(newRecord);
+ }
+
+ @Override
+ public void initialize(EventReporter reporter) throws IOException {
+ }
+
+ @Override
+ public List<ProvenanceEventRecord> getEvents(long firstRecordId, int maxRecords) throws IOException {
+ if (firstRecordId > records.size()) {
+ return Collections.emptyList();
+ }
+
+ return records.subList((int) firstRecordId, Math.min(records.size(), (int) (firstRecordId + maxRecords)));
+ }
+
+ @Override
+ public Long getMaxEventId() {
+ return Long.valueOf(records.size() - 1);
+ }
+
+ @Override
+ public QuerySubmission submitQuery(Query query) {
+ throw new UnsupportedOperationException("MockProvenanceEventRepository does not support querying");
+ }
+
+ @Override
+ public QuerySubmission retrieveQuerySubmission(String queryIdentifier) {
+ throw new UnsupportedOperationException("MockProvenanceEventRepository does not support querying");
+ }
+
+ @Override
+ public ComputeLineageSubmission submitLineageComputation(String flowFileUuid) {
+ throw new UnsupportedOperationException("MockProvenanceEventRepository does not support Lineage Computation");
+ }
+
+ @Override
+ public ComputeLineageSubmission retrieveLineageSubmission(String lineageIdentifier) {
+ throw new UnsupportedOperationException("MockProvenanceEventRepository does not support Lineage Computation");
+ }
+
+ @Override
+ public ProvenanceEventRecord getEvent(long id) throws IOException {
+ if (id > records.size()) {
+ return null;
+ }
+
+ return records.get((int) id);
+ }
+
+ @Override
+ public ComputeLineageSubmission submitExpandParents(long eventId) {
+ throw new UnsupportedOperationException("MockProvenanceEventRepository does not support Lineage Computation");
+ }
+
+ @Override
+ public ComputeLineageSubmission submitExpandChildren(long eventId) {
+ throw new UnsupportedOperationException("MockProvenanceEventRepository does not support Lineage Computation");
+ }
+
+ @Override
+ public void close() throws IOException {
+ }
+
+ @Override
+ public List<SearchableField> getSearchableFields() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public List<SearchableField> getSearchableAttributes() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public ProvenanceEventBuilder eventBuilder() {
+ return new StandardProvenanceEventRecord.Builder();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-mock/src/main/java/org/apache/nifi/reporting/BulletinFactory.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/main/java/org/apache/nifi/reporting/BulletinFactory.java b/nifi-mock/src/main/java/org/apache/nifi/reporting/BulletinFactory.java
new file mode 100644
index 0000000..221bed6
--- /dev/null
+++ b/nifi-mock/src/main/java/org/apache/nifi/reporting/BulletinFactory.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.reporting;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+public class BulletinFactory {
+
+ private static final AtomicLong currentId = new AtomicLong(0);
+
+ public static Bulletin createBulletin(final String groupId, final String sourceId, final String sourceName, final String category, final String severity, final String message) {
+ final Bulletin bulletin = new MockBulletin(currentId.getAndIncrement());
+ bulletin.setGroupId(groupId);
+ bulletin.setSourceId(sourceId);
+ bulletin.setSourceName(sourceName);
+ bulletin.setCategory(category);
+ bulletin.setLevel(severity);
+ bulletin.setMessage(message);
+ return bulletin;
+ }
+
+ public static Bulletin createBulletin(final String category, final String severity, final String message) {
+ final Bulletin bulletin = new MockBulletin(currentId.getAndIncrement());
+ bulletin.setCategory(category);
+ bulletin.setLevel(severity);
+ bulletin.setMessage(message);
+ return bulletin;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-mock/src/main/java/org/apache/nifi/reporting/MockBulletin.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/main/java/org/apache/nifi/reporting/MockBulletin.java b/nifi-mock/src/main/java/org/apache/nifi/reporting/MockBulletin.java
new file mode 100644
index 0000000..652fd51
--- /dev/null
+++ b/nifi-mock/src/main/java/org/apache/nifi/reporting/MockBulletin.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.reporting;
+
+public class MockBulletin extends Bulletin {
+
+ protected MockBulletin(long id) {
+ super(id);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-mock/src/main/java/org/apache/nifi/util/ControllerServiceConfiguration.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/ControllerServiceConfiguration.java b/nifi-mock/src/main/java/org/apache/nifi/util/ControllerServiceConfiguration.java
new file mode 100644
index 0000000..bd623ca
--- /dev/null
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/ControllerServiceConfiguration.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.controller.ControllerService;
+
+public class ControllerServiceConfiguration {
+
+ private final ControllerService service;
+ private final AtomicBoolean enabled = new AtomicBoolean(false);
+ private String annotationData;
+ private Map<PropertyDescriptor, String> properties = new HashMap<>();
+
+ public ControllerServiceConfiguration(final ControllerService service) {
+ this.service = service;
+ }
+
+ public ControllerService getService() {
+ return service;
+ }
+
+ public void setEnabled(final boolean enabled) {
+ this.enabled.set(enabled);
+ }
+
+ public boolean isEnabled() {
+ return this.enabled.get();
+ }
+
+ public void setProperties(final Map<PropertyDescriptor, String> props) {
+ this.properties = new HashMap<>(props);
+ }
+
+ public String getProperty(final PropertyDescriptor descriptor) {
+ final String value = properties.get(descriptor);
+ if (value == null) {
+ return descriptor.getDefaultValue();
+ } else {
+ return value;
+ }
+ }
+
+ public void setAnnotationData(final String annotationData) {
+ this.annotationData = annotationData;
+ }
+
+ public String getAnnotationData() {
+ return annotationData;
+ }
+
+ public Map<PropertyDescriptor, String> getProperties() {
+ return Collections.unmodifiableMap(properties);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-mock/src/main/java/org/apache/nifi/util/MockBulletinRepository.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/MockBulletinRepository.java b/nifi-mock/src/main/java/org/apache/nifi/util/MockBulletinRepository.java
new file mode 100644
index 0000000..04fc496
--- /dev/null
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/MockBulletinRepository.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import java.util.List;
+
+import org.apache.nifi.reporting.Bulletin;
+import org.apache.nifi.reporting.BulletinQuery;
+import org.apache.nifi.reporting.BulletinRepository;
+
+public class MockBulletinRepository implements BulletinRepository {
+
+ @Override
+ public void addBulletin(Bulletin bulletin) {
+ // TODO: Implement
+
+ }
+
+ @Override
+ public int getControllerBulletinCapacity() {
+ // TODO: Implement
+ return 0;
+ }
+
+ @Override
+ public int getComponentBulletinCapacity() {
+ // TODO: Implement
+ return 0;
+ }
+
+ @Override
+ public List<Bulletin> findBulletins(BulletinQuery bulletinQuery) {
+ // TODO: Implement
+ return null;
+ }
+
+ @Override
+ public List<Bulletin> findBulletinsForGroupBySource(String groupId) {
+ // TODO: Implement
+ return null;
+ }
+
+ @Override
+ public List<Bulletin> findBulletinsForGroupBySource(String groupId, int maxPerComponent) {
+ // TODO: Implement
+ return null;
+ }
+
+ @Override
+ public List<Bulletin> findBulletinsForController() {
+ // TODO: Implement
+ return null;
+ }
+
+ @Override
+ public List<Bulletin> findBulletinsForController(int max) {
+ // TODO: Implement
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-mock/src/main/java/org/apache/nifi/util/MockConfigurationContext.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/MockConfigurationContext.java b/nifi-mock/src/main/java/org/apache/nifi/util/MockConfigurationContext.java
new file mode 100644
index 0000000..c90e722
--- /dev/null
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/MockConfigurationContext.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.PropertyValue;
+import org.apache.nifi.controller.ConfigurationContext;
+import org.apache.nifi.controller.ControllerService;
+import org.apache.nifi.controller.ControllerServiceLookup;
+
+public class MockConfigurationContext implements ConfigurationContext {
+
+ private final Map<PropertyDescriptor, String> properties;
+ private final ControllerServiceLookup serviceLookup;
+ private final ControllerService service;
+
+ public MockConfigurationContext(final Map<PropertyDescriptor, String> properties, final ControllerServiceLookup serviceLookup) {
+ this(null, properties, serviceLookup);
+ }
+
+ public MockConfigurationContext(final ControllerService service, final Map<PropertyDescriptor, String> properties, final ControllerServiceLookup serviceLookup) {
+ this.service = service;
+ this.properties = properties;
+ this.serviceLookup = serviceLookup;
+ }
+
+ @Override
+ public PropertyValue getProperty(final PropertyDescriptor property) {
+ String value = properties.get(property);
+ if (value == null) {
+ value = getActualDescriptor(property).getDefaultValue();
+ }
+ return new MockPropertyValue(value, serviceLookup);
+ }
+
+ @Override
+ public Map<PropertyDescriptor, String> getProperties() {
+ return new HashMap<>(this.properties);
+ }
+
+ private PropertyDescriptor getActualDescriptor(final PropertyDescriptor property) {
+ if (service == null) {
+ return property;
+ }
+
+ final PropertyDescriptor resolved = service.getPropertyDescriptor(property.getName());
+ return resolved == null ? property : resolved;
+ }
+
+ @Override
+ public String getSchedulingPeriod() {
+ return "0 secs";
+ }
+
+ @Override
+ public Long getSchedulingPeriod(final TimeUnit timeUnit) {
+ return 0L;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-mock/src/main/java/org/apache/nifi/util/MockControllerServiceInitializationContext.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/MockControllerServiceInitializationContext.java b/nifi-mock/src/main/java/org/apache/nifi/util/MockControllerServiceInitializationContext.java
new file mode 100644
index 0000000..bff1d62
--- /dev/null
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/MockControllerServiceInitializationContext.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import org.apache.nifi.controller.ControllerService;
+import org.apache.nifi.controller.ControllerServiceInitializationContext;
+import org.apache.nifi.controller.ControllerServiceLookup;
+import org.apache.nifi.logging.ComponentLog;
+
+public class MockControllerServiceInitializationContext extends MockControllerServiceLookup implements ControllerServiceInitializationContext, ControllerServiceLookup {
+
+ private final String identifier;
+ private final ComponentLog logger;
+
+ public MockControllerServiceInitializationContext(final ControllerService controllerService, final String identifier) {
+ this(controllerService, identifier, new MockProcessorLog(identifier, controllerService));
+ }
+
+ public MockControllerServiceInitializationContext(final ControllerService controllerService, final String identifier, final ComponentLog logger) {
+ this.identifier = identifier;
+ this.logger = logger;
+ addControllerService(controllerService, identifier);
+ }
+
+ @Override
+ public String getIdentifier() {
+ return identifier;
+ }
+
+ @Override
+ public String getControllerServiceName(final String serviceIdentifier) {
+ return null;
+ }
+
+ @Override
+ public ControllerServiceLookup getControllerServiceLookup() {
+ return this;
+ }
+
+ @Override
+ public ComponentLog getLogger() {
+ return logger;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-mock/src/main/java/org/apache/nifi/util/MockControllerServiceLookup.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/MockControllerServiceLookup.java b/nifi-mock/src/main/java/org/apache/nifi/util/MockControllerServiceLookup.java
new file mode 100644
index 0000000..219ee24
--- /dev/null
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/MockControllerServiceLookup.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.nifi.controller.ControllerService;
+import org.apache.nifi.controller.ControllerServiceLookup;
+
+public abstract class MockControllerServiceLookup implements ControllerServiceLookup {
+
+ private final Map<String, ControllerServiceConfiguration> controllerServiceMap = new ConcurrentHashMap<>();
+
+ public ControllerServiceConfiguration addControllerService(final ControllerService service, final String identifier) {
+ final ControllerServiceConfiguration config = new ControllerServiceConfiguration(service);
+ controllerServiceMap.put(identifier, config);
+ return config;
+ }
+
+ public ControllerServiceConfiguration addControllerService(final ControllerService service) {
+ return addControllerService(service, service.getIdentifier());
+ }
+
+ public void removeControllerService(ControllerService service) {
+ final ControllerService canonical = getControllerService(service.getIdentifier());
+ if (canonical == null || canonical != service) {
+ throw new IllegalArgumentException("Controller Service " + service + " is not known");
+ }
+
+ controllerServiceMap.remove(service.getIdentifier());
+ }
+
+ protected void addControllerServices(final MockControllerServiceLookup other) {
+ this.controllerServiceMap.putAll(other.controllerServiceMap);
+ }
+
+ protected ControllerServiceConfiguration getConfiguration(final String identifier) {
+ return controllerServiceMap.get(identifier);
+ }
+
+ @Override
+ public ControllerService getControllerService(final String identifier) {
+ final ControllerServiceConfiguration status = controllerServiceMap.get(identifier);
+ return (status == null) ? null : status.getService();
+ }
+
+ @Override
+ public boolean isControllerServiceEnabled(final String serviceIdentifier) {
+ final ControllerServiceConfiguration status = controllerServiceMap.get(serviceIdentifier);
+ if (status == null) {
+ throw new IllegalArgumentException("No ControllerService exists with identifier " + serviceIdentifier);
+ }
+
+ return status.isEnabled();
+ }
+
+ @Override
+ public boolean isControllerServiceEnabled(final ControllerService service) {
+ return isControllerServiceEnabled(service.getIdentifier());
+ }
+
+ @Override
+ public boolean isControllerServiceEnabling(final String serviceIdentifier) {
+ return false;
+ }
+
+ @Override
+ public Set<String> getControllerServiceIdentifiers(final Class<? extends ControllerService> serviceType) {
+ final Set<String> ids = new HashSet<>();
+ for (final Map.Entry<String, ControllerServiceConfiguration> entry : controllerServiceMap.entrySet()) {
+ if (serviceType.isAssignableFrom(entry.getValue().getService().getClass())) {
+ ids.add(entry.getKey());
+ }
+ }
+ return ids;
+ }
+
+ @Override
+ public String getControllerServiceName(String serviceIdentifier) {
+ final ControllerServiceConfiguration status = controllerServiceMap.get(serviceIdentifier);
+ return status == null ? null : serviceIdentifier;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-mock/src/main/java/org/apache/nifi/util/MockEventAccess.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/MockEventAccess.java b/nifi-mock/src/main/java/org/apache/nifi/util/MockEventAccess.java
new file mode 100644
index 0000000..b5f6b11
--- /dev/null
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/MockEventAccess.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.nifi.controller.status.ProcessGroupStatus;
+import org.apache.nifi.provenance.ProvenanceEventRecord;
+import org.apache.nifi.provenance.ProvenanceEventRepository;
+import org.apache.nifi.reporting.EventAccess;
+
+public class MockEventAccess implements EventAccess {
+
+ private ProcessGroupStatus processGroupStatus;
+ private final List<ProvenanceEventRecord> provenanceRecords = new ArrayList<>();
+
+ public void setProcessGroupStatus(final ProcessGroupStatus status) {
+ this.processGroupStatus = status;
+ }
+
+ @Override
+ public ProcessGroupStatus getControllerStatus() {
+ return processGroupStatus;
+ }
+
+ @Override
+ public List<ProvenanceEventRecord> getProvenanceEvents(long firstEventId, int maxRecords) throws IOException {
+ if (firstEventId < 0 || maxRecords < 1) {
+ throw new IllegalArgumentException();
+ }
+
+ final List<ProvenanceEventRecord> records = new ArrayList<>();
+
+ for (final ProvenanceEventRecord record : provenanceRecords) {
+ if (record.getEventId() >= firstEventId) {
+ records.add(record);
+ if (records.size() >= maxRecords) {
+ return records;
+ }
+ }
+ }
+
+ return records;
+ }
+
+ public void addProvenanceEvent(final ProvenanceEventRecord record) {
+ this.provenanceRecords.add(record);
+ }
+
+ @Override
+ public ProvenanceEventRepository getProvenanceRepository() {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-mock/src/main/java/org/apache/nifi/util/MockFlowFile.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/MockFlowFile.java b/nifi-mock/src/main/java/org/apache/nifi/util/MockFlowFile.java
new file mode 100644
index 0000000..e9fb9d6
--- /dev/null
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/MockFlowFile.java
@@ -0,0 +1,277 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import java.io.BufferedInputStream;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+
+import org.junit.Assert;
+
+public class MockFlowFile implements FlowFile {
+
+ private final Map<String, String> attributes = new HashMap<>();
+
+ private final long id;
+ private final long entryDate;
+ private final Set<String> lineageIdentifiers = new HashSet<>();
+ private final long creationTime;
+ private boolean penalized = false;
+
+ private byte[] data = new byte[0];
+
+ public MockFlowFile(final long id) {
+ this.creationTime = System.nanoTime();
+ this.id = id;
+ entryDate = System.currentTimeMillis();
+ attributes.put(CoreAttributes.FILENAME.key(), String.valueOf(System.nanoTime()) + ".mockFlowFile");
+ attributes.put(CoreAttributes.PATH.key(), "target");
+
+ final String uuid = UUID.randomUUID().toString();
+ attributes.put(CoreAttributes.UUID.key(), uuid);
+ lineageIdentifiers.add(uuid);
+ }
+
+ public MockFlowFile(final long id, final FlowFile toCopy) {
+ this(id);
+ attributes.putAll(toCopy.getAttributes());
+ final byte[] dataToCopy = ((MockFlowFile) toCopy).data;
+ this.data = new byte[dataToCopy.length];
+ System.arraycopy(dataToCopy, 0, this.data, 0, dataToCopy.length);
+
+ lineageIdentifiers.addAll(toCopy.getLineageIdentifiers());
+ }
+
+ void setPenalized() {
+ this.penalized = true;
+ }
+
+ public long getCreationTime() {
+ return creationTime;
+ }
+
+ @Override
+ public Set<String> getLineageIdentifiers() {
+ return lineageIdentifiers;
+ }
+
+ @Override
+ public long getLineageStartDate() {
+ return entryDate;
+ }
+
+ @Override
+ public int compareTo(final FlowFile o) {
+ return getAttribute(CoreAttributes.UUID.key()).compareTo(o.getAttribute(CoreAttributes.UUID.key()));
+ }
+
+ @Override
+ public String getAttribute(final String attrName) {
+ return attributes.get(attrName);
+ }
+
+ @Override
+ public Map<String, String> getAttributes() {
+ return Collections.unmodifiableMap(attributes);
+ }
+
+ @Override
+ public long getEntryDate() {
+ return entryDate;
+ }
+
+ @Override
+ public long getId() {
+ return id;
+ }
+
+ @Override
+ public long getSize() {
+ return data.length;
+ }
+
+ void setData(final byte[] data) {
+ this.data = data;
+ }
+
+ byte[] getData() {
+ return this.data;
+ }
+
+ @Override
+ public boolean isPenalized() {
+ return penalized;
+ }
+
+ public void putAttributes(final Map<String, String> attrs) {
+ attributes.putAll(attrs);
+ }
+
+ public void removeAttributes(final Set<String> attrNames) {
+ for (final String attrName : attrNames) {
+ attributes.remove(attrName);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "FlowFile[" + id + "," + getAttribute(CoreAttributes.FILENAME.key()) + "," + getSize() + "B]";
+ }
+
+ @Override
+ public int hashCode() {
+ return (int) id;
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (obj == null) {
+ return false;
+ }
+ if (obj instanceof MockFlowFile) {
+ return ((MockFlowFile) obj).id == this.id;
+ }
+ return false;
+ }
+
+ public void assertAttributeExists(final String attributeName) {
+ Assert.assertTrue("Attribute " + attributeName + " does not exist", attributes.containsKey(attributeName));
+ }
+
+ public void assertAttributeNotExists(final String attributeName) {
+ Assert.assertFalse("Attribute " + attributeName + " not exists with value " + attributes.get(attributeName),
+ attributes.containsKey(attributeName));
+ }
+
+ public void assertAttributeEquals(final String attributeName, final String expectedValue) {
+ Assert.assertEquals(expectedValue, attributes.get(attributeName));
+ }
+
+ public void assertAttributeNotEquals(final String attributeName, final String expectedValue) {
+ Assert.assertNotSame(expectedValue, attributes.get(attributeName));
+ }
+
+ /**
+ * Asserts that the content of this FlowFile is the same as the content of
+ * the given file
+ *
+ * @param file to compare content against
+ * @throws IOException if fails doing IO during comparison
+ */
+ public void assertContentEquals(final File file) throws IOException {
+ assertContentEquals(file.toPath());
+ }
+
+ /**
+ * Asserts that the content of this FlowFile is the same as the content of
+ * the given path
+ *
+ * @param path where to find content to compare to
+ * @throws IOException if io error occurs while comparing content
+ */
+ public void assertContentEquals(final Path path) throws IOException {
+ try (final InputStream in = Files.newInputStream(path, StandardOpenOption.READ)) {
+ assertContentEquals(in);
+ }
+ }
+
+ /**
+ * Asserts that the content of this FlowFile is the same as the content of
+ * the given byte array
+ *
+ * @param data the data to compare
+ * @throws IOException if any ioe occurs while reading flowfile
+ */
+ public void assertContentEquals(final byte[] data) throws IOException {
+ try (final InputStream in = new ByteArrayInputStream(data)) {
+ assertContentEquals(in);
+ }
+ }
+
+ public void assertContentEquals(final String data) {
+ assertContentEquals(data, "UTF-8");
+ }
+
+ public void assertContentEquals(final String data, final String charset) {
+ assertContentEquals(data, Charset.forName(charset));
+ }
+
+ public void assertContentEquals(final String data, final Charset charset) {
+ final String value = new String(this.data, charset);
+ Assert.assertEquals(data, value);
+ }
+
+ /**
+ * Asserts that the content of this FlowFile is the same as the content of
+ * the given InputStream. This method closes the InputStream when it is
+ * finished.
+ *
+ * @param in the stream to source comparison data from
+ * @throws IOException if any issues reading from given source
+ */
+ public void assertContentEquals(final InputStream in) throws IOException {
+ int bytesRead = 0;
+ try (final BufferedInputStream buffered = new BufferedInputStream(in)) {
+ for (int i = 0; i < data.length; i++) {
+ final int fromStream = buffered.read();
+ if (fromStream < 0) {
+ Assert.fail("FlowFile content is " + data.length + " bytes but provided input is only " + bytesRead + " bytes");
+ }
+
+ if ((fromStream & 0xFF) != (data[i] & 0xFF)) {
+ Assert.fail("FlowFile content differs from input at byte " + bytesRead + " with input having value "
+ + (fromStream & 0xFF) + " and FlowFile having value " + (data[i] & 0xFF));
+ }
+
+ bytesRead++;
+ }
+
+ final int nextByte = buffered.read();
+ if (nextByte >= 0) {
+ Assert.fail("Contents of input and FlowFile were the same through byte " + data.length + "; however, FlowFile's content ended at this point, and input has more data");
+ }
+ }
+ }
+
+ /**
+ * @return a copy of the the contents of the FlowFile as a byte array
+ */
+ public byte[] toByteArray() {
+ return Arrays.copyOf(this.data, this.data.length);
+ }
+
+ @Override
+ public Long getLastQueueDate() {
+ return entryDate;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-mock/src/main/java/org/apache/nifi/util/MockFlowFileQueue.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/MockFlowFileQueue.java b/nifi-mock/src/main/java/org/apache/nifi/util/MockFlowFileQueue.java
new file mode 100644
index 0000000..775a1d5
--- /dev/null
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/MockFlowFileQueue.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import java.util.Collection;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.nifi.processor.QueueSize;
+
+public class MockFlowFileQueue {
+
+ private final BlockingQueue<MockFlowFile> queue;
+
+ private final ReadWriteLock rwLock = new ReentrantReadWriteLock();
+ private final Lock readLock = rwLock.readLock();
+ private final Lock writeLock = rwLock.writeLock();
+
+ public MockFlowFileQueue() {
+ queue = new LinkedBlockingQueue<>();
+ }
+
+ public void offer(final MockFlowFile flowFile) {
+ writeLock.lock();
+ try {
+ queue.offer(flowFile);
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ public MockFlowFile poll() {
+ writeLock.lock();
+ try {
+ return queue.poll();
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ public void addAll(final Collection<MockFlowFile> flowFiles) {
+ writeLock.lock();
+ try {
+ queue.addAll(flowFiles);
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ public QueueSize size() {
+ readLock.lock();
+ try {
+ final int count = queue.size();
+
+ long contentSize = 0L;
+ for (final MockFlowFile flowFile : queue) {
+ contentSize += flowFile.getSize();
+ }
+ return new QueueSize(count, contentSize);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ public boolean isEmpty() {
+ return size().getObjectCount() == 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessContext.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessContext.java b/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessContext.java
new file mode 100644
index 0000000..20a2f7c
--- /dev/null
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessContext.java
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import static java.util.Objects.requireNonNull;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+import org.apache.nifi.components.ConfigurableComponent;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.PropertyValue;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.controller.ControllerService;
+import org.apache.nifi.controller.ControllerServiceLookup;
+import org.apache.nifi.processor.Processor;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.SchedulingContext;
+import org.junit.Assert;
+
+public class MockProcessContext extends MockControllerServiceLookup implements SchedulingContext, ControllerServiceLookup {
+
+ private final ConfigurableComponent component;
+ private final Map<PropertyDescriptor, String> properties = new HashMap<>();
+
+ private String annotationData = null;
+ private boolean yieldCalled = false;
+ private boolean enableExpressionValidation = false;
+ private boolean allowExpressionValidation = true;
+
+ private volatile Set<Relationship> unavailableRelationships = new HashSet<>();
+
+ /**
+ * Creates a new MockProcessContext for the given Processor
+ *
+ * @param component being mocked
+ */
+ public MockProcessContext(final ConfigurableComponent component) {
+ this.component = Objects.requireNonNull(component);
+ }
+
+ public MockProcessContext(final ControllerService component, final MockProcessContext context) {
+ this(component);
+
+ try {
+ annotationData = context.getControllerServiceAnnotationData(component);
+ final Map<PropertyDescriptor, String> props = context.getControllerServiceProperties(component);
+ properties.putAll(props);
+
+ super.addControllerServices(context);
+ } catch (IllegalArgumentException e) {
+ // do nothing...the service is being loaded
+ }
+ }
+
+ @Override
+ public PropertyValue getProperty(final PropertyDescriptor descriptor) {
+ return getProperty(descriptor.getName());
+ }
+
+ @Override
+ public PropertyValue getProperty(final String propertyName) {
+ final PropertyDescriptor descriptor = component.getPropertyDescriptor(propertyName);
+ if (descriptor == null) {
+ return null;
+ }
+
+ final String setPropertyValue = properties.get(descriptor);
+ final String propValue = (setPropertyValue == null) ? descriptor.getDefaultValue() : setPropertyValue;
+ return new MockPropertyValue(propValue, this, (enableExpressionValidation && allowExpressionValidation) ? descriptor : null);
+ }
+
+ @Override
+ public PropertyValue newPropertyValue(final String rawValue) {
+ return new MockPropertyValue(rawValue, this);
+ }
+
+ public ValidationResult setProperty(final String propertyName, final String propertyValue) {
+ return setProperty(new PropertyDescriptor.Builder().name(propertyName).build(), propertyValue);
+ }
+
+ /**
+ * Updates the value of the property with the given PropertyDescriptor to
+ * the specified value IF and ONLY IF the value is valid according to the
+ * descriptor's validator. Otherwise, the property value is not updated. In
+ * either case, the ValidationResult is returned, indicating whether or not
+ * the property is valid
+ *
+ * @param descriptor of property to modify
+ * @param value new value
+ * @return result
+ */
+ public ValidationResult setProperty(final PropertyDescriptor descriptor, final String value) {
+ requireNonNull(descriptor);
+ requireNonNull(value, "Cannot set property to null value; if the intent is to remove the property, call removeProperty instead");
+ final PropertyDescriptor fullyPopulatedDescriptor = component.getPropertyDescriptor(descriptor.getName());
+
+ final ValidationResult result = fullyPopulatedDescriptor.validate(value, new MockValidationContext(this));
+ String oldValue = properties.put(fullyPopulatedDescriptor, value);
+ if (oldValue == null) {
+ oldValue = fullyPopulatedDescriptor.getDefaultValue();
+ }
+ if ((value == null && oldValue != null) || (value != null && !value.equals(oldValue))) {
+ component.onPropertyModified(fullyPopulatedDescriptor, oldValue, value);
+ }
+
+ return result;
+ }
+
+ public boolean removeProperty(final PropertyDescriptor descriptor) {
+ Objects.requireNonNull(descriptor);
+ final PropertyDescriptor fullyPopulatedDescriptor = component.getPropertyDescriptor(descriptor.getName());
+ String value = null;
+ if (!fullyPopulatedDescriptor.isRequired() && (value = properties.remove(fullyPopulatedDescriptor)) != null) {
+ component.onPropertyModified(fullyPopulatedDescriptor, value, null);
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public void yield() {
+ yieldCalled = true;
+ }
+
+ public boolean isYieldCalled() {
+ return yieldCalled;
+ }
+
+ public void addControllerService(final String serviceIdentifier, final ControllerService controllerService, final Map<PropertyDescriptor, String> properties, final String annotationData) {
+ requireNonNull(controllerService);
+ final ControllerServiceConfiguration config = addControllerService(controllerService);
+ config.setProperties(properties);
+ config.setAnnotationData(annotationData);
+ }
+
+ @Override
+ public int getMaxConcurrentTasks() {
+ return 1;
+ }
+
+ public void setAnnotationData(final String annotationData) {
+ this.annotationData = annotationData;
+ }
+
+ @Override
+ public String getAnnotationData() {
+ return annotationData;
+ }
+
+ @Override
+ public Map<PropertyDescriptor, String> getProperties() {
+ final List<PropertyDescriptor> supported = component.getPropertyDescriptors();
+ if (supported == null || supported.isEmpty()) {
+ return Collections.unmodifiableMap(properties);
+ } else {
+ final Map<PropertyDescriptor, String> props = new LinkedHashMap<>();
+ for (final PropertyDescriptor descriptor : supported) {
+ props.put(descriptor, null);
+ }
+ props.putAll(properties);
+ return props;
+ }
+ }
+
+ /**
+ * Validates the current properties, returning ValidationResults for any
+ * invalid properties. All processor defined properties will be validated.
+ * If they are not included in the in the purposed configuration, the
+ * default value will be used.
+ *
+ * @return Collection of validation result objects for any invalid findings
+ * only. If the collection is empty then the processor is valid. Guaranteed
+ * non-null
+ */
+ public Collection<ValidationResult> validate() {
+ return component.validate(new MockValidationContext(this));
+ }
+
+ public boolean isValid() {
+ for (final ValidationResult result : validate()) {
+ if (!result.isValid()) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ public void assertValid() {
+ final StringBuilder sb = new StringBuilder();
+ int failureCount = 0;
+
+ for (final ValidationResult result : validate()) {
+ if (!result.isValid()) {
+ sb.append(result.toString()).append("\n");
+ failureCount++;
+ }
+ }
+
+ if (failureCount > 0) {
+ Assert.fail("Processor has " + failureCount + " validation failures:\n" + sb.toString());
+ }
+ }
+
+ @Override
+ public String encrypt(final String unencrypted) {
+ return "enc{" + unencrypted + "}";
+ }
+
+ @Override
+ public String decrypt(final String encrypted) {
+ if (encrypted.startsWith("enc{") && encrypted.endsWith("}")) {
+ return encrypted.substring(4, encrypted.length() - 2);
+ }
+ return encrypted;
+ }
+
+ public void setValidateExpressionUsage(final boolean validate) {
+ allowExpressionValidation = validate;
+ }
+
+ public void enableExpressionValidation() {
+ enableExpressionValidation = true;
+ }
+
+ public void disableExpressionValidation() {
+ enableExpressionValidation = false;
+ }
+
+ Map<PropertyDescriptor, String> getControllerServiceProperties(final ControllerService controllerService) {
+ return super.getConfiguration(controllerService.getIdentifier()).getProperties();
+ }
+
+ String getControllerServiceAnnotationData(final ControllerService controllerService) {
+ return super.getConfiguration(controllerService.getIdentifier()).getAnnotationData();
+ }
+
+ @Override
+ public ControllerServiceLookup getControllerServiceLookup() {
+ return this;
+ }
+
+ @Override
+ public void leaseControllerService(final String identifier) {
+ }
+
+ public Set<Relationship> getAvailableRelationships() {
+ if (!(component instanceof Processor)) {
+ return Collections.emptySet();
+ }
+
+ final Set<Relationship> relationships = new HashSet<>(((Processor) component).getRelationships());
+ relationships.removeAll(unavailableRelationships);
+ return relationships;
+ }
+
+ public void setUnavailableRelationships(final Set<Relationship> relationships) {
+ this.unavailableRelationships = Collections.unmodifiableSet(new HashSet<>(relationships));
+ }
+
+ public Set<Relationship> getUnavailableRelationships() {
+ return unavailableRelationships;
+ }
+}
[18/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/asciidoc-mod.css
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/asciidoc-mod.css b/nifi-docs/src/main/asciidoc/asciidoc-mod.css
new file mode 100644
index 0000000..527bd3e
--- /dev/null
+++ b/nifi-docs/src/main/asciidoc/asciidoc-mod.css
@@ -0,0 +1,418 @@
+/* Asciidoctor default stylesheet | MIT License | http://asciidoctor.org */
+/* Copyright (C) 2012-2015 Dan Allen, Ryan Waldron and the Asciidoctor Project
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE. */
+/* Remove the comments around the @import statement below when using this as a custom stylesheet */
+@import "https://fonts.googleapis.com/css?family=Open+Sans:300,300italic,400,400italic,600,600italic%7CNoto+Serif:400,400italic,700,700italic%7CDroid+Sans+Mono:400";
+article,aside,details,figcaption,figure,footer,header,hgroup,main,nav,section,summary{display:block}
+audio,canvas,video{display:inline-block}
+audio:not([controls]){display:none;height:0}
+[hidden],template{display:none}
+script{display:none!important}
+html{font-family:sans-serif;-ms-text-size-adjust:100%;-webkit-text-size-adjust:100%}
+body{margin:0}
+a{background:transparent}
+a:focus{outline:thin dotted}
+a:active,a:hover{outline:0}
+h1{font-size:2em;margin:.67em 0}
+abbr[title]{border-bottom:1px dotted}
+b,strong{font-weight:bold}
+dfn{font-style:italic}
+hr{-moz-box-sizing:content-box;box-sizing:content-box;height:0}
+mark{background:#ff0;color:#000}
+code,kbd,pre,samp{font-family:monospace;font-size:1em}
+pre{white-space:pre-wrap}
+q{quotes:"\201C" "\201D" "\2018" "\2019"}
+small{font-size:80%}
+sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}
+sup{top:-.5em}
+sub{bottom:-.25em}
+img{border:0}
+svg:not(:root){overflow:hidden}
+figure{margin:0}
+fieldset{border:1px solid silver;margin:0 2px;padding:.35em .625em .75em}
+legend{border:0;padding:0}
+button,input,select,textarea{font-family:inherit;font-size:100%;margin:0}
+button,input{line-height:normal}
+button,select{text-transform:none}
+button,html input[type="button"],input[type="reset"],input[type="submit"]{-webkit-appearance:button;cursor:pointer}
+button[disabled],html input[disabled]{cursor:default}
+input[type="checkbox"],input[type="radio"]{box-sizing:border-box;padding:0}
+input[type="search"]{-webkit-appearance:textfield;-moz-box-sizing:content-box;-webkit-box-sizing:content-box;box-sizing:content-box}
+input[type="search"]::-webkit-search-cancel-button,input[type="search"]::-webkit-search-decoration{-webkit-appearance:none}
+button::-moz-focus-inner,input::-moz-focus-inner{border:0;padding:0}
+textarea{overflow:auto;vertical-align:top}
+table{border-collapse:collapse;border-spacing:0}
+*,*:before,*:after{-moz-box-sizing:border-box;-webkit-box-sizing:border-box;box-sizing:border-box}
+html,body{font-size:100%}
+body{background:#fff;color:rgba(0,0,0,.8);padding:0;margin:0;font-family:"Noto Serif","DejaVu Serif",serif;font-weight:400;font-style:normal;line-height:1;position:relative;cursor:auto}
+a:hover{cursor:pointer}
+img,object,embed{max-width:100%;height:auto}
+object,embed{height:100%}
+img{-ms-interpolation-mode:bicubic}
+#map_canvas img,#map_canvas embed,#map_canvas object,.map_canvas img,.map_canvas embed,.map_canvas object{max-width:none!important}
+.left{float:left!important}
+.right{float:right!important}
+.text-left{text-align:left!important}
+.text-right{text-align:right!important}
+.text-center{text-align:center!important}
+.text-justify{text-align:justify!important}
+.hide{display:none}
+.antialiased,body{-webkit-font-smoothing:antialiased}
+img{display:inline-block;vertical-align:middle}
+textarea{height:auto;min-height:50px}
+select{width:100%}
+p.lead,.paragraph.lead>p,#preamble>.sectionbody>.paragraph:first-of-type p{font-size:1.21875em;line-height:1.6}
+.subheader,.admonitionblock td.content>.title,.audioblock>.title,.exampleblock>.title,.imageblock>.title,.listingblock>.title,.literalblock>.title,.stemblock>.title,.openblock>.title,.paragraph>.title,.quoteblock>.title,table.tableblock>.title,.verseblock>.title,.videoblock>.title,.dlist>.title,.olist>.title,.ulist>.title,.qlist>.title,.hdlist>.title{line-height:1.45;color:#7a2518;font-weight:400;margin-top:0;margin-bottom:.25em}
+div,dl,dt,dd,ul,ol,li,h1,h2,h3,#toctitle,.sidebarblock>.content>.title,h4,h5,h6,pre,form,p,blockquote,th,td{margin:0;padding:0;direction:ltr}
+a{color:#2156a5;text-decoration:underline;line-height:inherit}
+a:hover,a:focus{color:#1d4b8f}
+a img{border:none}
+p{font-family:inherit;font-weight:400;font-size:1em;line-height:1.6;margin-bottom:1.25em;text-rendering:optimizeLegibility}
+p aside{font-size:.875em;line-height:1.35;font-style:italic}
+h1,h2,h3,#toctitle,.sidebarblock>.content>.title,h4,h5,h6{font-family:"Open Sans","DejaVu Sans",sans-serif;font-weight:300;font-style:normal;color:#ba3925;text-rendering:optimizeLegibility;margin-top:1em;margin-bottom:.5em;line-height:1.0125em}
+h1 small,h2 small,h3 small,#toctitle small,.sidebarblock>.content>.title small,h4 small,h5 small,h6 small{font-size:60%;color:#e99b8f;line-height:0}
+h1{font-size:2.125em}
+h2{font-size:1.6875em}
+h3,#toctitle,.sidebarblock>.content>.title{font-size:1.375em}
+h4,h5{font-size:1.125em}
+h6{font-size:1em}
+hr{border:solid #ddddd8;border-width:1px 0 0;clear:both;margin:1.25em 0 1.1875em;height:0}
+em,i{font-style:italic;line-height:inherit}
+strong,b{font-weight:bold;line-height:inherit}
+small{font-size:60%;line-height:inherit}
+code{font-family:"Droid Sans Mono","DejaVu Sans Mono",monospace;font-weight:400;color:rgba(0,0,0,.9);padding-right: 1px;}
+ul,ol,dl{font-size:1em;line-height:1.6;margin-bottom:1.25em;list-style-position:outside;font-family:inherit}
+ul,ol,ul.no-bullet,ol.no-bullet{margin-left:1.5em}
+ul li ul,ul li ol{margin-left:1.25em;margin-bottom:0;font-size:1em}
+ul.square li ul,ul.circle li ul,ul.disc li ul{list-style:inherit}
+ul.square{list-style-type:square}
+ul.circle{list-style-type:circle}
+ul.disc{list-style-type:disc}
+ul.no-bullet{list-style:none}
+ol li ul,ol li ol{margin-left:1.25em;margin-bottom:0}
+dl dt{margin-bottom:.3125em;font-weight:bold}
+dl dd{margin-bottom:1.25em}
+abbr,acronym{text-transform:uppercase;font-size:90%;color:rgba(0,0,0,.8);border-bottom:1px dotted #ddd;cursor:help}
+abbr{text-transform:none}
+blockquote{margin:0 0 1.25em;padding:.5625em 1.25em 0 1.1875em;border-left:1px solid #ddd}
+blockquote cite{display:block;font-size:.9375em;color:rgba(0,0,0,.6)}
+blockquote cite:before{content:"\2014 \0020"}
+blockquote cite a,blockquote cite a:visited{color:rgba(0,0,0,.6)}
+blockquote,blockquote p{line-height:1.6;color:rgba(0,0,0,.85)}
+@media only screen and (min-width:768px){h1,h2,h3,#toctitle,.sidebarblock>.content>.title,h4,h5,h6{line-height:1.2}
+h1{font-size:2.75em}
+h2{font-size:2.3125em}
+h3,#toctitle,.sidebarblock>.content>.title{font-size:1.6875em}
+h4{font-size:1.4375em}}table{background:#fff;margin-bottom:1.25em;border:solid 1px #dedede}
+table thead,table tfoot{background:#f7f8f7;font-weight:bold}
+table thead tr th,table thead tr td,table tfoot tr th,table tfoot tr td{padding:.5em .625em .625em;font-size:inherit;color:rgba(0,0,0,.8);text-align:left}
+table tr th,table tr td{padding:.5625em .625em;font-size:inherit;color:rgba(0,0,0,.8)}
+table tr.even,table tr.alt,table tr:nth-of-type(even){background:#f8f8f7}
+table thead tr th,table tfoot tr th,table tbody tr td,table tr td,table tfoot tr td{display:table-cell;line-height:1.6}
+h1,h2,h3,#toctitle,.sidebarblock>.content>.title,h4,h5,h6{line-height:1.2;word-spacing:-.05em}
+h1 strong,h2 strong,h3 strong,#toctitle strong,.sidebarblock>.content>.title strong,h4 strong,h5 strong,h6 strong{font-weight:400}
+.clearfix:before,.clearfix:after,.float-group:before,.float-group:after{content:" ";display:table}
+.clearfix:after,.float-group:after{clear:both}
+*:not(pre)>code{font-size:.9375em;font-style:normal!important;letter-spacing:0;word-spacing:-.15em;background-color:#f7f7f8;-webkit-border-radius:4px;border-radius:4px;line-height:1.45;text-rendering:optimizeSpeed}
+pre,pre>code{line-height:1.45;color:rgba(0,0,0,.9);font-family:"Droid Sans Mono","DejaVu Sans Mono",monospace;font-weight:400;text-rendering:optimizeSpeed}
+.keyseq{color:rgba(51,51,51,.8)}
+kbd{display:inline-block;color:rgba(0,0,0,.8);font-size:.75em;line-height:1.4;background-color:#f7f7f7;border:1px solid #ccc;-webkit-border-radius:3px;border-radius:3px;-webkit-box-shadow:0 1px 0 rgba(0,0,0,.2),0 0 0 .1em white inset;box-shadow:0 1px 0 rgba(0,0,0,.2),0 0 0 .1em #fff inset;margin:-.15em .15em 0 .15em;padding:.2em .6em .2em .5em;vertical-align:middle;white-space:nowrap}
+.keyseq kbd:first-child{margin-left:0}
+.keyseq kbd:last-child{margin-right:0}
+.menuseq,.menu{color:rgba(0,0,0,.8)}
+b.button:before,b.button:after{position:relative;top:-1px;font-weight:400}
+b.button:before{content:"[";padding:0 3px 0 2px}
+b.button:after{content:"]";padding:0 2px 0 3px}
+p a>code:hover{color:rgba(0,0,0,.9)}
+#header,#content,#footnotes,#footer{width:100%;margin-left:auto;margin-right:auto;margin-top:0;margin-bottom:0;max-width:62.5em;*zoom:1;position:relative;padding-left:.9375em;padding-right:.9375em}
+#header:before,#header:after,#content:before,#content:after,#footnotes:before,#footnotes:after,#footer:before,#footer:after{content:" ";display:table}
+#header:after,#content:after,#footnotes:after,#footer:after{clear:both}
+#content{margin-top:1.25em}
+#content:before{content:none}
+#header>h1:first-child{color:rgba(0,0,0,.85);margin-top:2.25rem;margin-bottom:0}
+#header>h1:first-child+#toc{margin-top:8px;border-top:1px solid #ddddd8}
+#header>h1:only-child,body.toc2 #header>h1:nth-last-child(2){border-bottom:1px solid #ddddd8;padding-bottom:8px}
+#header .details{border-bottom:1px solid #ddddd8;line-height:1.45;padding-top:.25em;padding-bottom:.25em;padding-left:.25em;color:rgba(0,0,0,.6);display:-ms-flexbox;display:-webkit-flex;display:flex;-ms-flex-flow:row wrap;-webkit-flex-flow:row wrap;flex-flow:row wrap}
+#header .details span:first-child{margin-left:-.125em}
+#header .details span.email a{color:rgba(0,0,0,.85)}
+#header .details br{display:none}
+#header .details br+span:before{content:"\00a0\2013\00a0"}
+#header .details br+span.author:before{content:"\00a0\22c5\00a0";color:rgba(0,0,0,.85)}
+#header .details br+span#revremark:before{content:"\00a0|\00a0"}
+#header #revnumber{text-transform:capitalize}
+#header #revnumber:after{content:"\00a0"}
+#content>h1:first-child:not([class]){color:rgba(0,0,0,.85);border-bottom:1px solid #ddddd8;padding-bottom:8px;margin-top:0;padding-top:1rem;margin-bottom:1.25rem}
+#toc{border-bottom:1px solid #efefed;padding-bottom:.5em}
+#toc>ul{margin-left:.125em}
+#toc ul.sectlevel0>li>a{font-style:italic}
+#toc ul.sectlevel0 ul.sectlevel1{margin:.5em 0}
+#toc ul{font-family:"Open Sans","DejaVu Sans",sans-serif;list-style-type:none}
+#toc a{text-decoration:none}
+#toc a:active{text-decoration:underline}
+#toctitle{color:#7a2518;font-size:1.2em}
+@media only screen and (min-width:768px){#toctitle{font-size:1.375em}
+body.toc2{padding-left:15em;padding-right:0}
+#toc.toc2{margin-top:0!important;background-color:#f8f8f7;position:fixed;width:15em;left:0;top:0;border-right:1px solid #efefed;border-top-width:0!important;border-bottom-width:0!important;z-index:1000;padding:1.25em 1em;height:100%;overflow:auto}
+#toc.toc2 #toctitle{margin-top:0;font-size:1.2em}
+#toc.toc2>ul{font-size:.9em;margin-bottom:0}
+#toc.toc2 ul ul{margin-left:0;padding-left:1em}
+#toc.toc2 ul.sectlevel0 ul.sectlevel1{padding-left:0;margin-top:.5em;margin-bottom:.5em}
+body.toc2.toc-right{padding-left:0;padding-right:15em}
+body.toc2.toc-right #toc.toc2{border-right-width:0;border-left:1px solid #efefed;left:auto;right:0}}@media only screen and (min-width:1280px){body.toc2{padding-left:20em;padding-right:0}
+#toc.toc2{width:20em}
+#toc.toc2 #toctitle{font-size:1.375em}
+#toc.toc2>ul{font-size:.95em}
+#toc.toc2 ul ul{padding-left:1.25em}
+body.toc2.toc-right{padding-left:0;padding-right:20em}}#content #toc{border-style:solid;border-width:1px;border-color:#e0e0dc;margin-bottom:1.25em;padding:1.25em;background:#f8f8f7;-webkit-border-radius:4px;border-radius:4px}
+#content #toc>:first-child{margin-top:0}
+#content #toc>:last-child{margin-bottom:0}
+#footer{max-width:100%;background-color:rgba(0,0,0,.8);padding:1.25em}
+#footer-text{color:rgba(255,255,255,.8);line-height:1.44}
+.sect1{padding-bottom:.625em}
+@media only screen and (min-width:768px){.sect1{padding-bottom:1.25em}}.sect1+.sect1{border-top:1px solid #efefed}
+#content h1>a.anchor,h2>a.anchor,h3>a.anchor,#toctitle>a.anchor,.sidebarblock>.content>.title>a.anchor,h4>a.anchor,h5>a.anchor,h6>a.anchor{position:absolute;z-index:1001;width:1.5ex;margin-left:-1.5ex;display:block;text-decoration:none!important;visibility:hidden;text-align:center;font-weight:400}
+#content h1>a.anchor:before,h2>a.anchor:before,h3>a.anchor:before,#toctitle>a.anchor:before,.sidebarblock>.content>.title>a.anchor:before,h4>a.anchor:before,h5>a.anchor:before,h6>a.anchor:before{content:"\00A7";font-size:.85em;display:block;padding-top:.1em}
+#content h1:hover>a.anchor,#content h1>a.anchor:hover,h2:hover>a.anchor,h2>a.anchor:hover,h3:hover>a.anchor,#toctitle:hover>a.anchor,.sidebarblock>.content>.title:hover>a.anchor,h3>a.anchor:hover,#toctitle>a.anchor:hover,.sidebarblock>.content>.title>a.anchor:hover,h4:hover>a.anchor,h4>a.anchor:hover,h5:hover>a.anchor,h5>a.anchor:hover,h6:hover>a.anchor,h6>a.anchor:hover{visibility:visible}
+#content h1>a.link,h2>a.link,h3>a.link,#toctitle>a.link,.sidebarblock>.content>.title>a.link,h4>a.link,h5>a.link,h6>a.link{color:#ba3925;text-decoration:none}
+#content h1>a.link:hover,h2>a.link:hover,h3>a.link:hover,#toctitle>a.link:hover,.sidebarblock>.content>.title>a.link:hover,h4>a.link:hover,h5>a.link:hover,h6>a.link:hover{color:#a53221}
+.audioblock,.imageblock,.literalblock,.listingblock,.stemblock,.videoblock{margin-bottom:1.25em}
+.admonitionblock td.content>.title,.audioblock>.title,.exampleblock>.title,.imageblock>.title,.listingblock>.title,.literalblock>.title,.stemblock>.title,.openblock>.title,.paragraph>.title,.quoteblock>.title,table.tableblock>.title,.verseblock>.title,.videoblock>.title,.dlist>.title,.olist>.title,.ulist>.title,.qlist>.title,.hdlist>.title{text-rendering:optimizeLegibility;text-align:left;font-family:"Noto Serif","DejaVu Serif",serif;font-size:1rem;font-style:italic}
+table.tableblock>caption.title{white-space:nowrap;overflow:visible;max-width:0}
+.paragraph.lead>p,#preamble>.sectionbody>.paragraph:first-of-type p{color:rgba(0,0,0,.85)}
+table.tableblock #preamble>.sectionbody>.paragraph:first-of-type p{font-size:inherit}
+.admonitionblock>table{border-collapse:separate;border:0;background:none;width:100%}
+.admonitionblock>table td.icon{text-align:center;width:80px}
+.admonitionblock>table td.icon img{max-width:none}
+.admonitionblock>table td.icon .title{font-weight:bold;font-family:"Open Sans","DejaVu Sans",sans-serif;text-transform:uppercase}
+.admonitionblock>table td.content{padding-left:1.125em;padding-right:1.25em;border-left:1px solid #ddddd8;color:rgba(0,0,0,.6)}
+.admonitionblock>table td.content>:last-child>:last-child{margin-bottom:0}
+.exampleblock>.content{border-style:solid;border-width:1px;border-color:#e6e6e6;margin-bottom:1.25em;padding:1.25em;background:#fff;-webkit-border-radius:4px;border-radius:4px}
+.exampleblock>.content>:first-child{margin-top:0}
+.exampleblock>.content>:last-child{margin-bottom:0}
+.sidebarblock{border-style:solid;border-width:1px;border-color:#e0e0dc;margin-bottom:1.25em;padding:1.25em;background:#f8f8f7;-webkit-border-radius:4px;border-radius:4px}
+.sidebarblock>:first-child{margin-top:0}
+.sidebarblock>:last-child{margin-bottom:0}
+.sidebarblock>.content>.title{color:#7a2518;margin-top:0;text-align:center}
+.exampleblock>.content>:last-child>:last-child,.exampleblock>.content .olist>ol>li:last-child>:last-child,.exampleblock>.content .ulist>ul>li:last-child>:last-child,.exampleblock>.content .qlist>ol>li:last-child>:last-child,.sidebarblock>.content>:last-child>:last-child,.sidebarblock>.content .olist>ol>li:last-child>:last-child,.sidebarblock>.content .ulist>ul>li:last-child>:last-child,.sidebarblock>.content .qlist>ol>li:last-child>:last-child{margin-bottom:0}
+.literalblock pre,.listingblock pre:not(.highlight),.listingblock pre[class="highlight"],.listingblock pre[class^="highlight "],.listingblock pre.CodeRay,.listingblock pre.prettyprint{background:#f7f7f8}
+.sidebarblock .literalblock pre,.sidebarblock .listingblock pre:not(.highlight),.sidebarblock .listingblock pre[class="highlight"],.sidebarblock .listingblock pre[class^="highlight "],.sidebarblock .listingblock pre.CodeRay,.sidebarblock .listingblock pre.prettyprint{background:#f2f1f1}
+.literalblock pre,.literalblock pre[class],.listingblock pre,.listingblock pre[class]{-webkit-border-radius:4px;border-radius:4px;word-wrap:break-word;padding:1em;font-size:.8125em}
+.literalblock pre.nowrap,.literalblock pre[class].nowrap,.listingblock pre.nowrap,.listingblock pre[class].nowrap{overflow-x:auto;white-space:pre;word-wrap:normal}
+@media only screen and (min-width:768px){.literalblock pre,.literalblock pre[class],.listingblock pre,.listingblock pre[class]{font-size:.90625em}}@media only screen and (min-width:1280px){.literalblock pre,.literalblock pre[class],.listingblock pre,.listingblock pre[class]{font-size:1em}}.literalblock.output pre{color:#f7f7f8;background-color:rgba(0,0,0,.9)}
+.listingblock pre.highlightjs{padding:0}
+.listingblock pre.highlightjs>code{padding:1em;-webkit-border-radius:4px;border-radius:4px}
+.listingblock pre.prettyprint{border-width:0}
+.listingblock>.content{position:relative}
+.listingblock code[data-lang]:before{display:none;content:attr(data-lang);position:absolute;font-size:.75em;top:.425rem;right:.5rem;line-height:1;text-transform:uppercase;color:#999}
+.listingblock:hover code[data-lang]:before{display:block}
+.listingblock.terminal pre .command:before{content:attr(data-prompt);padding-right:.5em;color:#999}
+.listingblock.terminal pre .command:not([data-prompt]):before{content:"$"}
+table.pyhltable{border-collapse:separate;border:0;margin-bottom:0;background:none}
+table.pyhltable td{vertical-align:top;padding-top:0;padding-bottom:0}
+table.pyhltable td.code{padding-left:.75em;padding-right:0}
+pre.pygments .lineno,table.pyhltable td:not(.code){color:#999;padding-left:0;padding-right:.5em;border-right:1px solid #ddddd8}
+pre.pygments .lineno{display:inline-block;margin-right:.25em}
+table.pyhltable .linenodiv{background:none!important;padding-right:0!important}
+.quoteblock{margin:0 1em 1.25em 1.5em;display:table}
+.quoteblock>.title{margin-left:-1.5em;margin-bottom:.75em}
+.quoteblock blockquote,.quoteblock blockquote p{color:rgba(0,0,0,.85);font-size:1.15rem;line-height:1.75;word-spacing:.1em;letter-spacing:0;font-style:italic;text-align:justify}
+.quoteblock blockquote{margin:0;padding:0;border:0}
+.quoteblock blockquote:before{content:"\201c";float:left;font-size:2.75em;font-weight:bold;line-height:.6em;margin-left:-.6em;color:#7a2518;text-shadow:0 1px 2px rgba(0,0,0,.1)}
+.quoteblock blockquote>.paragraph:last-child p{margin-bottom:0}
+.quoteblock .attribution{margin-top:.5em;margin-right:.5ex;text-align:right}
+.quoteblock .quoteblock{margin-left:0;margin-right:0;padding:.5em 0;border-left:3px solid rgba(0,0,0,.6)}
+.quoteblock .quoteblock blockquote{padding:0 0 0 .75em}
+.quoteblock .quoteblock blockquote:before{display:none}
+.verseblock{margin:0 1em 1.25em 1em}
+.verseblock pre{font-family:"Open Sans","DejaVu Sans",sans;font-size:1.15rem;color:rgba(0,0,0,.85);font-weight:300;text-rendering:optimizeLegibility}
+.verseblock pre strong{font-weight:400}
+.verseblock .attribution{margin-top:1.25rem;margin-left:.5ex}
+.quoteblock .attribution,.verseblock .attribution{font-size:.9375em;line-height:1.45;font-style:italic}
+.quoteblock .attribution br,.verseblock .attribution br{display:none}
+.quoteblock .attribution cite,.verseblock .attribution cite{display:block;letter-spacing:-.05em;color:rgba(0,0,0,.6)}
+.quoteblock.abstract{margin:0 0 1.25em 0;display:block}
+.quoteblock.abstract blockquote,.quoteblock.abstract blockquote p{text-align:left;word-spacing:0}
+.quoteblock.abstract blockquote:before,.quoteblock.abstract blockquote p:first-of-type:before{display:none}
+table.tableblock{max-width:100%;border-collapse:separate}
+table.tableblock td>.paragraph:last-child p>p:last-child,table.tableblock th>p:last-child,table.tableblock td>p:last-child{margin-bottom:0}
+table.spread{width:100%}
+table.tableblock,th.tableblock,td.tableblock{border:0 solid #dedede}
+table.grid-all th.tableblock,table.grid-all td.tableblock{border-width:0 1px 1px 0}
+table.grid-all tfoot>tr>th.tableblock,table.grid-all tfoot>tr>td.tableblock{border-width:1px 1px 0 0}
+table.grid-cols th.tableblock,table.grid-cols td.tableblock{border-width:0 1px 0 0}
+table.grid-all *>tr>.tableblock:last-child,table.grid-cols *>tr>.tableblock:last-child{border-right-width:0}
+table.grid-rows th.tableblock,table.grid-rows td.tableblock{border-width:0 0 1px 0}
+table.grid-all tbody>tr:last-child>th.tableblock,table.grid-all tbody>tr:last-child>td.tableblock,table.grid-all thead:last-child>tr>th.tableblock,table.grid-rows tbody>tr:last-child>th.tableblock,table.grid-rows tbody>tr:last-child>td.tableblock,table.grid-rows thead:last-child>tr>th.tableblock{border-bottom-width:0}
+table.grid-rows tfoot>tr>th.tableblock,table.grid-rows tfoot>tr>td.tableblock{border-width:1px 0 0 0}
+table.frame-all{border-width:1px}
+table.frame-sides{border-width:0 1px}
+table.frame-topbot{border-width:1px 0}
+th.halign-left,td.halign-left{text-align:left}
+th.halign-right,td.halign-right{text-align:right}
+th.halign-center,td.halign-center{text-align:center}
+th.valign-top,td.valign-top{vertical-align:top}
+th.valign-bottom,td.valign-bottom{vertical-align:bottom}
+th.valign-middle,td.valign-middle{vertical-align:middle}
+table thead th,table tfoot th{font-weight:bold}
+tbody tr th{display:table-cell;line-height:1.6;background:#f7f8f7}
+tbody tr th,tbody tr th p,tfoot tr th,tfoot tr th p{color:rgba(0,0,0,.8);font-weight:bold}
+p.tableblock>code:only-child{background:none;padding:0}
+p.tableblock{font-size:1em}
+td>div.verse{white-space:pre}
+ol{margin-left:1.75em}
+ul li ol{margin-left:1.5em}
+dl dd{margin-left:1.125em}
+dl dd:last-child,dl dd:last-child>:last-child{margin-bottom:0}
+ol>li p,ul>li p,ul dd,ol dd,.olist .olist,.ulist .ulist,.ulist .olist,.olist .ulist{margin-bottom:.625em}
+ul.unstyled,ol.unnumbered,ul.checklist,ul.none{list-style-type:none}
+ul.unstyled,ol.unnumbered,ul.checklist{margin-left:.625em}
+ul.checklist li>p:first-child>.fa-square-o:first-child,ul.checklist li>p:first-child>.fa-check-square-o:first-child{width:1em;font-size:.85em}
+ul.checklist li>p:first-child>input[type="checkbox"]:first-child{width:1em;position:relative;top:1px}
+ul.inline{margin:0 auto .625em auto;margin-left:-1.375em;margin-right:0;padding:0;list-style:none;overflow:hidden}
+ul.inline>li{list-style:none;float:left;margin-left:1.375em;display:block}
+ul.inline>li>*{display:block}
+.unstyled dl dt{font-weight:400;font-style:normal}
+ol.arabic{list-style-type:decimal}
+ol.decimal{list-style-type:decimal-leading-zero}
+ol.loweralpha{list-style-type:lower-alpha}
+ol.upperalpha{list-style-type:upper-alpha}
+ol.lowerroman{list-style-type:lower-roman}
+ol.upperroman{list-style-type:upper-roman}
+ol.lowergreek{list-style-type:lower-greek}
+.hdlist>table,.colist>table{border:0;background:none}
+.hdlist>table>tbody>tr,.colist>table>tbody>tr{background:none}
+td.hdlist1{padding-right:.75em;font-weight:bold}
+td.hdlist1,td.hdlist2{vertical-align:top}
+.literalblock+.colist,.listingblock+.colist{margin-top:-.5em}
+.colist>table tr>td:first-of-type{padding:0 .75em;line-height:1}
+.colist>table tr>td:last-of-type{padding:.25em 0}
+.thumb,.th{line-height:0;display:inline-block;border:solid 4px #fff;-webkit-box-shadow:0 0 0 1px #ddd;box-shadow:0 0 0 1px #ddd}
+.imageblock.left,.imageblock[style*="float: left"]{margin:.25em .625em 1.25em 0}
+.imageblock.right,.imageblock[style*="float: right"]{margin:.25em 0 1.25em .625em}
+.imageblock>.title{margin-bottom:0}
+.imageblock.thumb,.imageblock.th{border-width:6px}
+.imageblock.thumb>.title,.imageblock.th>.title{padding:0 .125em}
+.image.left,.image.right{margin-top:.25em;margin-bottom:.25em;display:inline-block;line-height:0}
+.image.left{margin-right:.625em}
+.image.right{margin-left:.625em}
+a.image{text-decoration:none}
+span.footnote,span.footnoteref{vertical-align:super;font-size:.875em}
+span.footnote a,span.footnoteref a{text-decoration:none}
+span.footnote a:active,span.footnoteref a:active{text-decoration:underline}
+#footnotes{padding-top:.75em;padding-bottom:.75em;margin-bottom:.625em}
+#footnotes hr{width:20%;min-width:6.25em;margin:-.25em 0 .75em 0;border-width:1px 0 0 0}
+#footnotes .footnote{padding:0 .375em;line-height:1.3;font-size:.875em;margin-left:1.2em;text-indent:-1.2em;margin-bottom:.2em}
+#footnotes .footnote a:first-of-type{font-weight:bold;text-decoration:none}
+#footnotes .footnote:last-of-type{margin-bottom:0}
+#content #footnotes{margin-top:-.625em;margin-bottom:0;padding:.75em 0}
+.gist .file-data>table{border:0;background:#fff;width:100%;margin-bottom:0}
+.gist .file-data>table td.line-data{width:99%}
+div.unbreakable{page-break-inside:avoid}
+.big{font-size:larger}
+.small{font-size:smaller}
+.underline{text-decoration:underline}
+.overline{text-decoration:overline}
+.line-through{text-decoration:line-through}
+.aqua{color:#00bfbf}
+.aqua-background{background-color:#00fafa}
+.black{color:#000}
+.black-background{background-color:#000}
+.blue{color:#0000bf}
+.blue-background{background-color:#0000fa}
+.fuchsia{color:#bf00bf}
+.fuchsia-background{background-color:#fa00fa}
+.gray{color:#606060}
+.gray-background{background-color:#7d7d7d}
+.green{color:#006000}
+.green-background{background-color:#007d00}
+.lime{color:#00bf00}
+.lime-background{background-color:#00fa00}
+.maroon{color:#600000}
+.maroon-background{background-color:#7d0000}
+.navy{color:#000060}
+.navy-background{background-color:#00007d}
+.olive{color:#606000}
+.olive-background{background-color:#7d7d00}
+.purple{color:#600060}
+.purple-background{background-color:#7d007d}
+.red{color:#bf0000}
+.red-background{background-color:#fa0000}
+.silver{color:#909090}
+.silver-background{background-color:#bcbcbc}
+.teal{color:#006060}
+.teal-background{background-color:#007d7d}
+.white{color:#bfbfbf}
+.white-background{background-color:#fafafa}
+.yellow{color:#bfbf00}
+.yellow-background{background-color:#fafa00}
+span.icon>.fa{cursor:default}
+.admonitionblock td.icon [class^="fa icon-"]{font-size:2.5em;text-shadow:1px 1px 2px rgba(0,0,0,.5);cursor:default}
+.admonitionblock td.icon .icon-note:before{content:"\f05a";color:#19407c}
+.admonitionblock td.icon .icon-tip:before{content:"\f0eb";text-shadow:1px 1px 2px rgba(155,155,0,.8);color:#111}
+.admonitionblock td.icon .icon-warning:before{content:"\f071";color:#bf6900}
+.admonitionblock td.icon .icon-caution:before{content:"\f06d";color:#bf3400}
+.admonitionblock td.icon .icon-important:before{content:"\f06a";color:#bf0000}
+.conum[data-value]{display:inline-block;color:#fff!important;background-color:rgba(0,0,0,.8);-webkit-border-radius:100px;border-radius:100px;text-align:center;font-size:.75em;width:1.67em;height:1.67em;line-height:1.67em;font-family:"Open Sans","DejaVu Sans",sans-serif;font-style:normal;font-weight:bold}
+.conum[data-value] *{color:#fff!important}
+.conum[data-value]+b{display:none}
+.conum[data-value]:after{content:attr(data-value)}
+pre .conum[data-value]{position:relative;top:-.125em}
+b.conum *{color:inherit!important}
+.conum:not([data-value]):empty{display:none}
+h1,h2{letter-spacing:-.01em}
+dt,th.tableblock,td.content{text-rendering:optimizeLegibility}
+p,td.content{letter-spacing:-.01em}
+p strong,td.content strong{letter-spacing:-.005em}
+p,blockquote,dt,td.content{font-size:1.0625rem}
+p{margin-bottom:1.25rem}
+.sidebarblock p,.sidebarblock dt,.sidebarblock td.content,p.tableblock{font-size:1em}
+.exampleblock>.content{background-color:#fffef7;border-color:#e0e0dc;-webkit-box-shadow:0 1px 4px #e0e0dc;box-shadow:0 1px 4px #e0e0dc}
+.print-only{display:none!important}
+@media print{@page{margin:1.25cm .75cm}
+*{-webkit-box-shadow:none!important;box-shadow:none!important;text-shadow:none!important}
+a{color:inherit!important;text-decoration:underline!important}
+a.bare,a[href^="#"],a[href^="mailto:"]{text-decoration:none!important}
+a[href^="http:"]:not(.bare):after,a[href^="https:"]:not(.bare):after{content:"(" attr(href) ")";display:inline-block;font-size:.875em;padding-left:.25em}
+abbr[title]:after{content:" (" attr(title) ")"}
+pre,blockquote,tr,img{page-break-inside:avoid}
+thead{display:table-header-group}
+img{max-width:100%!important}
+p,blockquote,dt,td.content{font-size:1em;orphans:3;widows:3}
+h2,h3,#toctitle,.sidebarblock>.content>.title{page-break-after:avoid}
+#toc,.sidebarblock,.exampleblock>.content{background:none!important}
+#toc{border-bottom:1px solid #ddddd8!important;padding-bottom:0!important}
+.sect1{padding-bottom:0!important}
+.sect1+.sect1{border:0!important}
+#header>h1:first-child{margin-top:1.25rem}
+body.book #header{text-align:center}
+body.book #header>h1:first-child{border:0!important;margin:2.5em 0 1em 0}
+body.book #header .details{border:0!important;display:block;padding:0!important}
+body.book #header .details span:first-child{margin-left:0!important}
+body.book #header .details br{display:block}
+body.book #header .details br+span:before{content:none!important}
+body.book #toc{border:0!important;text-align:left!important;padding:0!important;margin:0!important}
+body.book #toc,body.book #preamble,body.book h1.sect0,body.book .sect1>h2{page-break-before:always}
+.listingblock code[data-lang]:before{display:block}
+#footer{background:none!important;padding:0 .9375em}
+#footer-text{color:rgba(0,0,0,.6)!important;font-size:.9em}
+.hide-on-print{display:none!important}
+.print-only{display:block!important}
+.hide-for-print{display:none!important}
+.show-for-print{display:inherit!important}}
\ No newline at end of file
[49/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnShutdown.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnShutdown.java b/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnShutdown.java
new file mode 100644
index 0000000..44098ff
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnShutdown.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.annotation.lifecycle;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import org.apache.nifi.controller.ConfigurationContext;
+import org.apache.nifi.processor.ProcessContext;
+
+/**
+ * <p>
+ * Marker annotation a {@link org.apache.nifi.processor.Processor Processor},
+ * {@link org.apache.nifi.controller.ControllerService ControllerService}, or
+ * {@link org.apache.nifi.reporting.ReportingTask ReportingTask} implementation
+ * can use to indicate a method should be called whenever the flow is being
+ * shutdown. This will be called at most once for each component in a JVM
+ * lifetime. It is not, however, guaranteed that this method will be called on
+ * shutdown, as the service may be killed suddenly.
+ * </p>
+ *
+ * <p>
+ * Methods with this annotation are permitted to take either 0 or 1 argument. If
+ * an argument is used, it must be of type {@link ConfigurationContext} if the
+ * component is a ReportingTask or Controller Service, or of type
+ * {@link ProcessContext} if the component is a Processor.
+ * </p>
+ *
+ */
+@Documented
+@Target({ElementType.METHOD})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+public @interface OnShutdown {
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnStopped.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnStopped.java b/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnStopped.java
new file mode 100644
index 0000000..622f158
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnStopped.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.annotation.lifecycle;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import org.apache.nifi.controller.ConfigurationContext;
+import org.apache.nifi.processor.ProcessContext;
+
+/**
+ * <p>
+ * Marker annotation a {@link org.apache.nifi.processor.Processor Processor} or
+ * {@link org.apache.nifi.reporting.ReportingTask ReportingTask} implementation
+ * can use to indicate that a method should be called whenever the component is
+ * no longer scheduled to run. Methods marked with this annotation will be
+ * invoked each time the component is stopped and will be invoked only after the
+ * last thread has returned from the <code>onTrigger</code> method.
+ * </p>
+ *
+ * <p>
+ * This means that the thread executing in this method will be the only thread
+ * executing in any part of the Processor. However, since other threads may
+ * later execute other parts of the code, member variables must still be
+ * protected appropriately. However, access to multiple variables need not be
+ * atomic.
+ * </p>
+ *
+ * <p>
+ * To indicate that a method should be called immediately when a component is no
+ * longer scheduled to run (as opposed to after all threads have returned from
+ * the <code>onTrigger</code> method), see the {@link OnUnscheduled} annotation.
+ * </p>
+ *
+ * <p>
+ * Methods with this annotation are permitted to take either 0 or 1 argument. If
+ * an argument is used, it must be of type {@link ConfigurationContext} if the
+ * component is a ReportingTask or of type {@link ProcessContext} if the
+ * component is a Processor.
+ * </p>
+ *
+ */
+@Documented
+@Target({ElementType.METHOD})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+public @interface OnStopped {
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnUnscheduled.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnUnscheduled.java b/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnUnscheduled.java
new file mode 100644
index 0000000..b7d5027
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnUnscheduled.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.annotation.lifecycle;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * <p>
+ * Marker annotation a {@link org.apache.nifi.processor.Processor Processor} or
+ * {@link org.apache.nifi.reporting.ReportingTask ReportingTask} should be
+ * called whenever the component is no longer scheduled to run. Methods marked
+ * with this annotation will be invoked each time the framework is notified to
+ * stop scheduling the component. This method is invoked as other threads are
+ * potentially running. To invoke a method after all threads have finished
+ * processing, see the {@link OnStopped} annotation.
+ * </p>
+ *
+ * <p>
+ * Methods using this annotation must take either 0 arguments or a single
+ * argument.
+ * </p>
+ *
+ * <p>
+ * If using 1 argument and the component using the annotation is a Processor,
+ * that argument must be of type
+ * {@link org.apache.nifi.processor.ProcessContext ProcessContext}.
+ * </p>
+ *
+ * <p>
+ * If using 1 argument and the component using the annotation is a Reporting
+ * Task, that argument must be of type
+ * {@link org.apache.nifi.controller.ConfigurationContext ConfigurationContext}.
+ * </p>
+ */
+@Documented
+@Target({ElementType.METHOD})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+public @interface OnUnscheduled {
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/annotation/notification/OnPrimaryNodeStateChange.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/annotation/notification/OnPrimaryNodeStateChange.java b/nifi-api/src/main/java/org/apache/nifi/annotation/notification/OnPrimaryNodeStateChange.java
new file mode 100644
index 0000000..4ea2170
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/annotation/notification/OnPrimaryNodeStateChange.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.annotation.notification;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * <p>
+ * Marker annotation that a component can use to indicate that a method should be
+ * called whenever the state of the Primary Node in a cluster has changed.
+ * </p>
+ *
+ * <p>
+ * Methods with this annotation should take either no arguments or one argument of type
+ * {@link PrimaryNodeState}. The {@link PrimaryNodeState} provides context about what changed
+ * so that the component can take appropriate action.
+ * </p>
+ */
+@Documented
+@Target({ElementType.METHOD})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+public @interface OnPrimaryNodeStateChange {
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/annotation/notification/PrimaryNodeState.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/annotation/notification/PrimaryNodeState.java b/nifi-api/src/main/java/org/apache/nifi/annotation/notification/PrimaryNodeState.java
new file mode 100644
index 0000000..0d65b65
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/annotation/notification/PrimaryNodeState.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.annotation.notification;
+
+/**
+ * Represents a state change that occurred for the Primary Node of a NiFi cluster.
+ */
+public enum PrimaryNodeState {
+ /**
+ * The node receiving this state has been elected the Primary Node of the NiFi cluster.
+ */
+ ELECTED_PRIMARY_NODE,
+
+ /**
+ * The node receiving this state was the Primary Node but has now had its Primary Node
+ * role revoked.
+ */
+ PRIMARY_NODE_REVOKED;
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/authorization/Authority.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/authorization/Authority.java b/nifi-api/src/main/java/org/apache/nifi/authorization/Authority.java
new file mode 100644
index 0000000..4502c11
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/authorization/Authority.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.authorization;
+
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+/**
+ * Authorities that can be assigned to NiFi users.
+ */
+public enum Authority {
+
+ ROLE_MONITOR,
+ ROLE_DFM,
+ ROLE_ADMIN,
+ ROLE_PROVENANCE,
+ ROLE_PROXY,
+ ROLE_NIFI;
+
+ /**
+ * @param rawAuthority string form of authority
+ * @return the matching role or null if the specified role does not match
+ * any roles
+ */
+ public static Authority valueOfAuthority(String rawAuthority) {
+ Authority desiredAuthority = null;
+
+ for (Authority authority : values()) {
+ if (authority.toString().equals(rawAuthority)) {
+ desiredAuthority = authority;
+ break;
+ }
+ }
+
+ return desiredAuthority;
+ }
+
+ /**
+ * @return the string value of each authority
+ */
+ public static Set<String> getRawAuthorities() {
+ Set<String> authorities = new LinkedHashSet<>();
+ for (Authority authority : values()) {
+ authorities.add(authority.toString());
+ }
+ return authorities;
+ }
+
+ public static Set<String> convertAuthorities(Set<Authority> authorities) {
+ if (authorities == null) {
+ throw new IllegalArgumentException("No authorities have been specified.");
+ }
+
+ // convert the set
+ Set<String> rawAuthorities = new HashSet<>(authorities.size());
+ for (Authority authority : authorities) {
+ rawAuthorities.add(authority.toString());
+ }
+ return rawAuthorities;
+ }
+
+ public static EnumSet<Authority> convertRawAuthorities(Set<String> rawAuthorities) {
+ if (rawAuthorities == null) {
+ throw new IllegalArgumentException("No authorities have been specified.");
+ }
+
+ // convert the set
+ EnumSet<Authority> authorities = EnumSet.noneOf(Authority.class);
+ for (String rawAuthority : rawAuthorities) {
+ Authority authority = Authority.valueOfAuthority(rawAuthority);
+ if (authority != null) {
+ authorities.add(authority);
+ }
+ }
+ return authorities;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/authorization/AuthorityProvider.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/authorization/AuthorityProvider.java b/nifi-api/src/main/java/org/apache/nifi/authorization/AuthorityProvider.java
new file mode 100644
index 0000000..723ec33
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/authorization/AuthorityProvider.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.authorization;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.nifi.authorization.exception.AuthorityAccessException;
+import org.apache.nifi.authorization.exception.IdentityAlreadyExistsException;
+import org.apache.nifi.authorization.exception.ProviderCreationException;
+import org.apache.nifi.authorization.exception.ProviderDestructionException;
+import org.apache.nifi.authorization.exception.UnknownIdentityException;
+
+/**
+ * This class allows clients to retrieve the authorities for a given DN.
+ */
+public interface AuthorityProvider {
+
+ /**
+ * @param dn of the user
+ * @return whether the user with the specified DN is known to this authority
+ * provider. It is not necessary for the user to have any authorities
+ */
+ boolean doesDnExist(String dn) throws AuthorityAccessException;
+
+ /**
+ * Get the authorities for the specified user. If the specified user exists
+ * but does not have any authorities, an empty set should be returned.
+ *
+ * @param dn of the user to lookup
+ * @return the authorities for the specified user. If the specified user
+ * exists but does not have any authorities, an empty set should be returned
+ * @throws UnknownIdentityException if identity is not known
+ * @throws AuthorityAccessException if unable to access authorities
+ */
+ Set<Authority> getAuthorities(String dn) throws UnknownIdentityException, AuthorityAccessException;
+
+ /**
+ * Sets the specified authorities for the specified user.
+ *
+ * @param dn the specified user
+ * @param authorities the new authorities for the user
+ * @throws UnknownIdentityException if identity is not known
+ * @throws AuthorityAccessException if unable to access authorities
+ */
+ void setAuthorities(String dn, Set<Authority> authorities) throws UnknownIdentityException, AuthorityAccessException;
+
+ /**
+ * Gets the users for the specified authority.
+ *
+ * @param authority for which to determine membership of
+ * @return all users with the specified authority
+ * @throws AuthorityAccessException if unable to access authorities
+ */
+ Set<String> getUsers(Authority authority) throws AuthorityAccessException;
+
+ /**
+ * Revokes the specified user. Its up to the implementor to determine the
+ * semantics of revocation.
+ *
+ * @param dn the dn of the user
+ * @throws UnknownIdentityException if the user is not known
+ * @throws AuthorityAccessException if unable to access the authorities
+ */
+ void revokeUser(String dn) throws UnknownIdentityException, AuthorityAccessException;
+
+ /**
+ * Add the specified user.
+ *
+ * @param dn of the user
+ * @param group Optional
+ * @throws UnknownIdentityException if the user is not known
+ * @throws AuthorityAccessException if unable to access the authorities
+ */
+ void addUser(String dn, String group) throws IdentityAlreadyExistsException, AuthorityAccessException;
+
+ /**
+ * Gets the group for the specified user. Return null if the user does not
+ * belong to a group.
+ *
+ * @param dn the user
+ * @return the group of the given user
+ * @throws UnknownIdentityException if the user is not known
+ * @throws AuthorityAccessException if unable to access the authorities
+ */
+ String getGroupForUser(String dn) throws UnknownIdentityException, AuthorityAccessException;
+
+ /**
+ * Revokes all users for a specified group. Its up to the implementor to
+ * determine the semantics of revocation.
+ *
+ * @param group to revoke the users of
+ * @throws UnknownIdentityException if the user is not known
+ * @throws AuthorityAccessException if unable to access the authorities
+ */
+ void revokeGroup(String group) throws UnknownIdentityException, AuthorityAccessException;
+
+ /**
+ * Adds the specified users to the specified group.
+ *
+ * @param dn the set of users to add to the group
+ * @param group to add users to
+ * @throws UnknownIdentityException if the user is not known
+ * @throws AuthorityAccessException if unable to access the authorities
+ */
+ void setUsersGroup(Set<String> dn, String group) throws UnknownIdentityException, AuthorityAccessException;
+
+ /**
+ * Ungroups the specified user.
+ *
+ * @param dn of the user
+ * @throws UnknownIdentityException if the user is not known
+ * @throws AuthorityAccessException if unable to access the authorities
+ */
+ void ungroupUser(String dn) throws UnknownIdentityException, AuthorityAccessException;
+
+ /**
+ * Ungroups the specified group. Since the semantics of revocation is up to
+ * the implementor, this method should do nothing if the specified group
+ * does not exist. If an admin revoked this group before calling ungroup, it
+ * may or may not exist.
+ *
+ * @param group to ungroup
+ * @throws AuthorityAccessException if unable to access the authorities
+ */
+ void ungroup(String group) throws AuthorityAccessException;
+
+ /**
+ * Determines whether the user in the specified dnChain should be able to
+ * download the content for the flowfile with the specified attributes.
+ *
+ * The first dn in the chain is the end user that the request was issued on
+ * behalf of. The subsequent dn's in the chain represent entities proxying
+ * the user's request with the last being the proxy that sent the current
+ * request.
+ *
+ * @param dnChain of the user
+ * @param attributes of the flowfile being requested
+ * @return the authorization result
+ * @throws UnknownIdentityException if the user is not known
+ * @throws AuthorityAccessException if unable to access the authorities
+ */
+ DownloadAuthorization authorizeDownload(List<String> dnChain, Map<String, String> attributes) throws UnknownIdentityException, AuthorityAccessException;
+
+ /**
+ * Called immediately after instance creation for implementers to perform
+ * additional setup
+ *
+ * @param initializationContext in which to initialize
+ */
+ void initialize(AuthorityProviderInitializationContext initializationContext) throws ProviderCreationException;
+
+ /**
+ * Called to configure the AuthorityProvider.
+ *
+ * @param configurationContext at the time of configuration
+ * @throws ProviderCreationException for any issues configuring the provider
+ */
+ void onConfigured(AuthorityProviderConfigurationContext configurationContext) throws ProviderCreationException;
+
+ /**
+ * Called immediately before instance destruction for implementers to
+ * release resources.
+ *
+ * @throws ProviderDestructionException If pre-destruction fails.
+ */
+ void preDestruction() throws ProviderDestructionException;
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/authorization/AuthorityProviderConfigurationContext.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/authorization/AuthorityProviderConfigurationContext.java b/nifi-api/src/main/java/org/apache/nifi/authorization/AuthorityProviderConfigurationContext.java
new file mode 100644
index 0000000..c1ba5df
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/authorization/AuthorityProviderConfigurationContext.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.authorization;
+
+import java.util.Map;
+
+/**
+ *
+ */
+public interface AuthorityProviderConfigurationContext {
+
+ /**
+ * @return identifier for the authority provider
+ */
+ String getIdentifier();
+
+ /**
+ * Retrieves all properties the component currently understands regardless
+ * of whether a value has been set for them or not. If no value is present
+ * then its value is null and thus any registered default for the property
+ * descriptor applies.
+ *
+ * @return Map of all properties
+ */
+ Map<String, String> getProperties();
+
+ /**
+ * @param property to lookup the descriptor and value of
+ * @return the value the component currently understands for the given
+ * PropertyDescriptor. This method does not substitute default
+ * PropertyDescriptor values, so the value returned will be null if not set
+ */
+ String getProperty(String property);
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/authorization/AuthorityProviderInitializationContext.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/authorization/AuthorityProviderInitializationContext.java b/nifi-api/src/main/java/org/apache/nifi/authorization/AuthorityProviderInitializationContext.java
new file mode 100644
index 0000000..7b2f89f
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/authorization/AuthorityProviderInitializationContext.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.authorization;
+
+/**
+ *
+ */
+public interface AuthorityProviderInitializationContext {
+
+ public String getIdentifier();
+
+ public AuthorityProviderLookup getAuthorityProviderLookup();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/authorization/AuthorityProviderLookup.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/authorization/AuthorityProviderLookup.java b/nifi-api/src/main/java/org/apache/nifi/authorization/AuthorityProviderLookup.java
new file mode 100644
index 0000000..dc30967
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/authorization/AuthorityProviderLookup.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.authorization;
+
+/**
+ *
+ */
+public interface AuthorityProviderLookup {
+
+ AuthorityProvider getAuthorityProvider(String identifier);
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/authorization/DownloadAuthorization.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/authorization/DownloadAuthorization.java b/nifi-api/src/main/java/org/apache/nifi/authorization/DownloadAuthorization.java
new file mode 100644
index 0000000..416f3cf
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/authorization/DownloadAuthorization.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.authorization;
+
+/**
+ * Represents a decision whether authorization is granted to download content.
+ */
+public class DownloadAuthorization {
+
+ private static enum Result {
+
+ Approved,
+ Denied;
+ }
+
+ private static final DownloadAuthorization APPROVED = new DownloadAuthorization(Result.Approved, null);
+
+ private final Result result;
+ private final String explanation;
+
+ /**
+ * Creates a new DownloadAuthorization with the specified result and
+ * explanation.
+ *
+ * @param result of the authorization
+ * @param explanation for the authorization attempt
+ */
+ private DownloadAuthorization(Result result, String explanation) {
+ if (Result.Denied.equals(result) && explanation == null) {
+ throw new IllegalArgumentException("An explanation is required when the download request is denied.");
+ }
+
+ this.result = result;
+ this.explanation = explanation;
+ }
+
+ /**
+ * @return Whether or not the download request is approved
+ */
+ public boolean isApproved() {
+ return Result.Approved.equals(result);
+ }
+
+ /**
+ * @return If the download request is denied, the reason why. Null otherwise
+ */
+ public String getExplanation() {
+ return explanation;
+ }
+
+ /**
+ * @return a new approved DownloadAuthorization
+ */
+ public static DownloadAuthorization approved() {
+ return APPROVED;
+ }
+
+ /**
+ * Creates a new denied DownloadAuthorization with the specified
+ * explanation.
+ *
+ * @param explanation for why it was denied
+ * @return a new denied DownloadAuthorization with the specified explanation
+ * @throws IllegalArgumentException if explanation is null
+ */
+ public static DownloadAuthorization denied(String explanation) {
+ return new DownloadAuthorization(Result.Denied, explanation);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/authorization/annotation/AuthorityProviderContext.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/authorization/annotation/AuthorityProviderContext.java b/nifi-api/src/main/java/org/apache/nifi/authorization/annotation/AuthorityProviderContext.java
new file mode 100644
index 0000000..5ac2af7
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/authorization/annotation/AuthorityProviderContext.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.authorization.annotation;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ *
+ *
+ */
+@Documented
+@Target({ElementType.FIELD, ElementType.METHOD})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+public @interface AuthorityProviderContext {
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/authorization/exception/AuthorityAccessException.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/authorization/exception/AuthorityAccessException.java b/nifi-api/src/main/java/org/apache/nifi/authorization/exception/AuthorityAccessException.java
new file mode 100644
index 0000000..be64767
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/authorization/exception/AuthorityAccessException.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.authorization.exception;
+
+/**
+ * Represents the case when the DN could not be confirmed because it was unable
+ * to access the data store.
+ */
+public class AuthorityAccessException extends RuntimeException {
+
+ public AuthorityAccessException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public AuthorityAccessException(String message) {
+ super(message);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/authorization/exception/IdentityAlreadyExistsException.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/authorization/exception/IdentityAlreadyExistsException.java b/nifi-api/src/main/java/org/apache/nifi/authorization/exception/IdentityAlreadyExistsException.java
new file mode 100644
index 0000000..ba80b6e
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/authorization/exception/IdentityAlreadyExistsException.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.authorization.exception;
+
+/**
+ * Represents the case when the user identity already exists.
+ */
+public class IdentityAlreadyExistsException extends RuntimeException {
+
+ public IdentityAlreadyExistsException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public IdentityAlreadyExistsException(String message) {
+ super(message);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/authorization/exception/ProviderCreationException.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/authorization/exception/ProviderCreationException.java b/nifi-api/src/main/java/org/apache/nifi/authorization/exception/ProviderCreationException.java
new file mode 100644
index 0000000..24ac793
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/authorization/exception/ProviderCreationException.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.authorization.exception;
+
+/**
+ * Represents the exceptional case when an AuthorityProvider fails instantiated.
+ *
+ */
+public class ProviderCreationException extends RuntimeException {
+
+ public ProviderCreationException() {
+ }
+
+ public ProviderCreationException(String msg) {
+ super(msg);
+ }
+
+ public ProviderCreationException(Throwable cause) {
+ super(cause);
+ }
+
+ public ProviderCreationException(String msg, Throwable cause) {
+ super(msg, cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/authorization/exception/ProviderDestructionException.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/authorization/exception/ProviderDestructionException.java b/nifi-api/src/main/java/org/apache/nifi/authorization/exception/ProviderDestructionException.java
new file mode 100644
index 0000000..985d3fb
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/authorization/exception/ProviderDestructionException.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.authorization.exception;
+
+/**
+ * Represents the exceptional case when an AuthorityProvider fails destruction.
+ *
+ */
+public class ProviderDestructionException extends RuntimeException {
+
+ public ProviderDestructionException() {
+ }
+
+ public ProviderDestructionException(String msg) {
+ super(msg);
+ }
+
+ public ProviderDestructionException(Throwable cause) {
+ super(cause);
+ }
+
+ public ProviderDestructionException(String msg, Throwable cause) {
+ super(msg, cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/authorization/exception/UnknownIdentityException.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/authorization/exception/UnknownIdentityException.java b/nifi-api/src/main/java/org/apache/nifi/authorization/exception/UnknownIdentityException.java
new file mode 100644
index 0000000..2ada1c7
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/authorization/exception/UnknownIdentityException.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.authorization.exception;
+
+/**
+ * Represents the case when an identity cannot be confirmed.
+ */
+public class UnknownIdentityException extends RuntimeException {
+
+ public UnknownIdentityException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public UnknownIdentityException(String message) {
+ super(message);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/components/AbstractConfigurableComponent.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/components/AbstractConfigurableComponent.java b/nifi-api/src/main/java/org/apache/nifi/components/AbstractConfigurableComponent.java
new file mode 100644
index 0000000..1d4e0c2
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/components/AbstractConfigurableComponent.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.components;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+public abstract class AbstractConfigurableComponent implements ConfigurableComponent {
+
+ /**
+ * Allows subclasses to perform their own validation on the already set
+ * properties. Since each property is validated as it is set this allows
+ * validation of groups of properties together. Default return is an empty
+ * set.
+ *
+ * This method will be called only when it has been determined that all
+ * property values are valid according to their corresponding
+ * PropertyDescriptor's validators.
+ *
+ * @param validationContext provides a mechanism for obtaining externally
+ * managed values, such as property values and supplies convenience methods
+ * for operating on those values
+ *
+ * @return Collection of ValidationResult objects that will be added to any
+ * other validation findings - may be null
+ */
+ protected Collection<ValidationResult> customValidate(final ValidationContext validationContext) {
+ return Collections.emptySet();
+ }
+
+ /**
+ * @param descriptorName to lookup the descriptor
+ * @return a PropertyDescriptor for the name specified that is fully
+ * populated
+ */
+ @Override
+ public final PropertyDescriptor getPropertyDescriptor(final String descriptorName) {
+ final PropertyDescriptor specDescriptor = new PropertyDescriptor.Builder().name(descriptorName).build();
+ return getPropertyDescriptor(specDescriptor);
+ }
+
+ private PropertyDescriptor getPropertyDescriptor(final PropertyDescriptor specDescriptor) {
+ PropertyDescriptor descriptor = null;
+ //check if property supported
+ final List<PropertyDescriptor> supportedDescriptors = getSupportedPropertyDescriptors();
+ if (supportedDescriptors != null) {
+ for (final PropertyDescriptor desc : supportedDescriptors) { //find actual descriptor
+ if (specDescriptor.equals(desc)) {
+ return desc;
+ }
+ }
+ }
+
+ descriptor = getSupportedDynamicPropertyDescriptor(specDescriptor.getName());
+ if (descriptor != null && !descriptor.isDynamic()) {
+ descriptor = new PropertyDescriptor.Builder().fromPropertyDescriptor(descriptor).dynamic(true).build();
+ }
+
+ if (descriptor == null) {
+ descriptor = new PropertyDescriptor.Builder().fromPropertyDescriptor(specDescriptor).addValidator(Validator.INVALID).dynamic(true).build();
+ }
+ return descriptor;
+ }
+
+ @Override
+ public final Collection<ValidationResult> validate(final ValidationContext context) {
+ // goes through supported properties
+ final Collection<ValidationResult> results = new ArrayList<>();
+ final List<PropertyDescriptor> supportedDescriptors = getSupportedPropertyDescriptors();
+ if (null != supportedDescriptors) {
+ for (final PropertyDescriptor descriptor : supportedDescriptors) {
+ String value = context.getProperty(descriptor).getValue();
+ if (value == null) {
+ value = descriptor.getDefaultValue();
+ }
+ if (value == null && descriptor.isRequired()) {
+ results.add(new ValidationResult.Builder().valid(false).input(null).subject(descriptor.getName()).explanation(descriptor.getName() + " is required").build());
+ continue;
+ } else if (value == null) {
+ continue;
+ }
+
+ final ValidationResult result = descriptor.validate(value, context);
+ if (!result.isValid()) {
+ results.add(result);
+ }
+ }
+ }
+
+ // validate any dynamic properties
+ for (final Map.Entry<PropertyDescriptor, String> entry : context.getProperties().entrySet()) {
+ final PropertyDescriptor descriptor = entry.getKey();
+ final String value = entry.getValue();
+
+ if (supportedDescriptors != null && !supportedDescriptors.contains(descriptor)) {
+ final ValidationResult result = descriptor.validate(value, context);
+ if (!result.isValid()) {
+ results.add(result);
+ }
+ }
+ }
+
+ // only run customValidate if regular validation is successful. This allows Processor developers to not have to check
+ // if values are null or invalid so that they can focus only on the interaction between the properties, etc.
+ if (results.isEmpty()) {
+ final Collection<ValidationResult> customResults = customValidate(context);
+ if (null != customResults) {
+ for (final ValidationResult result : customResults) {
+ if (!result.isValid()) {
+ results.add(result);
+ }
+ }
+ }
+ }
+
+ return results;
+ }
+
+ /**
+ * Hook method allowing subclasses to eagerly react to a configuration
+ * change for the given property descriptor. As an alternative to using this
+ * method a processor may simply get the latest value whenever it needs it
+ * and if necessary lazily evaluate it.
+ *
+ * @param descriptor of the modified property
+ * @param oldValue non-null property value (previous)
+ * @param newValue the new property value or if null indicates the property
+ * was removed
+ */
+ @Override
+ public void onPropertyModified(final PropertyDescriptor descriptor, final String oldValue, final String newValue) {
+ }
+
+ /**
+ * <p>
+ * Used to allow subclasses to determine what PropertyDescriptor if any to
+ * use when a property is requested for which a descriptor is not already
+ * registered. By default this method simply returns a null descriptor. By
+ * overriding this method processor implementations can support dynamic
+ * properties since this allows them to register properties on demand. It is
+ * acceptable for a dynamically generated property to indicate it is
+ * required so long as it is understood it is only required once set.
+ * Dynamic properties by definition cannot be required until used.</p>
+ *
+ * <p>
+ * This method should be side effect free in the subclasses in terms of how
+ * often it is called for a given property name because there is guarantees
+ * how often it will be called for a given property name.</p>
+ *
+ * <p>
+ * Default is null.
+ *
+ * @param propertyDescriptorName used to lookup if any property descriptors exist for that name
+ * @return new property descriptor if supported
+ */
+ protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) {
+ return null;
+ }
+
+ /**
+ * Allows subclasses to register which property descriptor objects are
+ * supported. Default return is an empty set.
+ *
+ * @return PropertyDescriptor objects this processor currently supports
+ */
+ @SuppressWarnings("unchecked")
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return Collections.EMPTY_LIST;
+ }
+
+ @Override
+ public final List<PropertyDescriptor> getPropertyDescriptors() {
+ final List<PropertyDescriptor> supported = getSupportedPropertyDescriptors();
+ return supported == null ? Collections.<PropertyDescriptor>emptyList() : new ArrayList<>(supported);
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (!(obj instanceof ConfigurableComponent)) {
+ return false;
+ }
+
+ final ConfigurableComponent other = (ConfigurableComponent) obj;
+ return getIdentifier().equals(other.getIdentifier());
+ }
+
+ @Override
+ public int hashCode() {
+ return 235 + getIdentifier().hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + "[id=" + getIdentifier() + "]";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/components/AllowableValue.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/components/AllowableValue.java b/nifi-api/src/main/java/org/apache/nifi/components/AllowableValue.java
new file mode 100644
index 0000000..a480961
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/components/AllowableValue.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.components;
+
+import java.util.Objects;
+
+/**
+ * <p>
+ * Represents a valid value for a {@link PropertyDescriptor}
+ * </p>
+ */
+public class AllowableValue {
+
+ private final String value;
+ private final String displayName;
+ private final String description;
+
+ /**
+ * Constructs a new AllowableValue with the given value and and the same
+ * display name and no description.
+ *
+ * @param value that is allowed
+ */
+ public AllowableValue(final String value) {
+ this(value, value);
+ }
+
+ /**
+ * Constructs a new AllowableValue with the given value and display name and
+ * no description
+ *
+ * @param value that is allowed
+ * @param displayName to display for the value
+ *
+ * @throws NullPointerException if either argument is null
+ */
+ public AllowableValue(final String value, final String displayName) {
+ this(value, displayName, null);
+ }
+
+ /**
+ * Constructs a new AllowableValue with the given value, display name, and
+ * description
+ *
+ * @param value that is valid
+ * @param displayName to show for the value
+ * @param description of the value
+ *
+ * @throws NullPointerException if identifier or value is null
+ */
+ public AllowableValue(final String value, final String displayName, final String description) {
+ this.value = Objects.requireNonNull(value);
+ this.displayName = Objects.requireNonNull(displayName);
+ this.description = description;
+ }
+
+ /**
+ * @return the value of this AllowableValue
+ */
+ public String getValue() {
+ return value;
+ }
+
+ /**
+ * @return a human-readable name for this AllowableValue
+ */
+ public String getDisplayName() {
+ return displayName;
+ }
+
+ /**
+ * @return a description for this value, or <code>null</code> if no
+ * description was provided
+ */
+ public String getDescription() {
+ return description;
+ }
+
+ /**
+ * @return true if <code>this</code> is equal to <code>obj</code> of <code>obj</code> is the
+ * same object as <code>this</code> or if <code>obj</code> is an instance of
+ * <code>AllowableValue</code> and both have the same value, or if
+ * <code>obj</code> is a String and is equal to
+ * {@link #getValue() this.getValue()}.
+ */
+ @Override
+ public boolean equals(final Object obj) {
+ if (obj == this) {
+ return true;
+ }
+
+ if (obj instanceof AllowableValue) {
+ final AllowableValue other = (AllowableValue) obj;
+ return (this.value.equals(other.getValue()));
+ } else if (obj instanceof String) {
+ return this.value.equals(obj);
+ }
+
+ return false;
+ }
+
+ /**
+ * @return based solely off of the value
+ */
+ @Override
+ public int hashCode() {
+ return 23984731 + 17 * value.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return value;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/components/ConfigurableComponent.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/components/ConfigurableComponent.java b/nifi-api/src/main/java/org/apache/nifi/components/ConfigurableComponent.java
new file mode 100644
index 0000000..8b56f48
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/components/ConfigurableComponent.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.components;
+
+import java.util.Collection;
+import java.util.List;
+
+public interface ConfigurableComponent {
+
+ /**
+ * Validates a set of properties, returning ValidationResults for any
+ * invalid properties. All defined properties will be validated. If they are
+ * not included in the in the purposed configuration, the default value will
+ * be used.
+ *
+ * @param context of validation
+ * @return Collection of validation result objects for any invalid findings
+ * only. If the collection is empty then the component is valid. Guaranteed
+ * non-null
+ */
+ Collection<ValidationResult> validate(ValidationContext context);
+
+ /**
+ * @param name to lookup the descriptor
+ * @return the PropertyDescriptor with the given name, if it exists;
+ * otherwise, returns <code>null</code>
+ */
+ PropertyDescriptor getPropertyDescriptor(String name);
+
+ /**
+ * Hook method allowing subclasses to eagerly react to a configuration
+ * change for the given property descriptor. This method will be invoked
+ * regardless of property validity. As an alternative to using this method,
+ * a component may simply get the latest value whenever it needs it and if
+ * necessary lazily evaluate it. Any throwable that escapes this method will
+ * simply be ignored.
+ *
+ * @param descriptor the descriptor for the property being modified
+ * @param oldValue the value that was previously set, or null if no value
+ * was previously set for this property
+ * @param newValue the new property value or if null indicates the property
+ * was removed
+ */
+ void onPropertyModified(PropertyDescriptor descriptor, String oldValue, String newValue);
+
+ /**
+ * Returns a {@link List} of all {@link PropertyDescriptor}s that this
+ * component supports.
+ *
+ * @return PropertyDescriptor objects this component currently supports
+ */
+ List<PropertyDescriptor> getPropertyDescriptors();
+
+ /**
+ * @return the unique identifier that the framework assigned to this
+ * component
+ */
+ String getIdentifier();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/components/PropertyDescriptor.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/components/PropertyDescriptor.java b/nifi-api/src/main/java/org/apache/nifi/components/PropertyDescriptor.java
new file mode 100644
index 0000000..acda6c4
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/components/PropertyDescriptor.java
@@ -0,0 +1,556 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.components;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.nifi.controller.ControllerService;
+
+/**
+ * An immutable object for holding information about a type of processor
+ * property.
+ *
+ */
+public final class PropertyDescriptor implements Comparable<PropertyDescriptor> {
+
+ public static final PropertyDescriptor NULL_DESCRIPTOR = new PropertyDescriptor.Builder().name("").build();
+
+ /**
+ * The proper name for the property. This is the primary mechanism of
+ * comparing equality.
+ */
+ private final String name;
+
+ /**
+ * The name that should be displayed to user when referencing this property
+ */
+ private final String displayName;
+
+ /**
+ * And explanation of the meaning of the given property. This description is
+ * meant to be displayed to a user or simply provide a mechanism of
+ * documenting intent.
+ */
+ private final String description;
+ /**
+ * The default value for this property
+ */
+ private final String defaultValue;
+ /**
+ * The allowable values for this property. If empty then the allowable
+ * values are not constrained
+ */
+ private final List<AllowableValue> allowableValues;
+ /**
+ * Determines whether the property is required for this processor
+ */
+ private final boolean required;
+ /**
+ * indicates that the value for this property should be considered sensitive
+ * and protected whenever stored or represented
+ */
+ private final boolean sensitive;
+ /**
+ * indicates whether this property well-known for this processor or is
+ * user-defined
+ */
+ private final boolean dynamic;
+ /**
+ * indicates whether or not this property supports the Attribute Expression
+ * Language
+ */
+ private final boolean expressionLanguageSupported;
+
+ /**
+ * the interface of the {@link ControllerService} that this Property refers
+ * to; will be null if not identifying a ControllerService.
+ */
+ private final Class<? extends ControllerService> controllerServiceDefinition;
+
+ /**
+ * The validators that should be used whenever an attempt is made to set
+ * this property value. Any allowable values specified will be checked first
+ * and any validators specified will be ignored.
+ */
+ private final List<Validator> validators;
+
+ protected PropertyDescriptor(final Builder builder) {
+ this.displayName = builder.displayName == null ? builder.name : builder.displayName;
+ this.name = builder.name;
+ this.description = builder.description;
+ this.defaultValue = builder.defaultValue;
+ this.allowableValues = builder.allowableValues;
+ this.required = builder.required;
+ this.sensitive = builder.sensitive;
+ this.dynamic = builder.dynamic;
+ this.expressionLanguageSupported = builder.expressionLanguageSupported;
+ this.controllerServiceDefinition = builder.controllerServiceDefinition;
+ this.validators = new ArrayList<>(builder.validators);
+ }
+
+ @Override
+ public int compareTo(final PropertyDescriptor o) {
+ if (o == null) {
+ return -1;
+ }
+ return getName().compareTo(o.getName());
+ }
+
+ /**
+ * Validates the given input against this property descriptor's validator.
+ * If this descriptor has a set of allowable values then the given value is
+ * only checked against the allowable values.
+ *
+ * @param input the value to validate
+ * @param context the context of validation
+ * @return the result of validating the input
+ */
+ public ValidationResult validate(final String input, final ValidationContext context) {
+ ValidationResult lastResult = Validator.INVALID.validate(this.name, input, context);
+ if (allowableValues != null && !allowableValues.isEmpty()) {
+ final ConstrainedSetValidator csValidator = new ConstrainedSetValidator(allowableValues);
+ final ValidationResult csResult = csValidator.validate(this.name, input, context);
+ if (csResult.isValid()) {
+ lastResult = csResult;
+ } else {
+ return csResult;
+ }
+ }
+
+ // if the property descriptor identifies a Controller Service, validate that the ControllerService exists, is of the correct type, and is valid
+ if (controllerServiceDefinition != null) {
+ final Set<String> validIdentifiers = context.getControllerServiceLookup().getControllerServiceIdentifiers(controllerServiceDefinition);
+ if (validIdentifiers != null && validIdentifiers.contains(input)) {
+ final ControllerService controllerService = context.getControllerServiceLookup().getControllerService(input);
+ if (!context.isValidationRequired(controllerService)) {
+ return new ValidationResult.Builder()
+ .input(input)
+ .subject(getName())
+ .valid(true)
+ .build();
+ }
+
+ final String serviceId = controllerService.getIdentifier();
+ if (!context.getControllerServiceLookup().isControllerServiceEnabled(serviceId)
+ && !context.getControllerServiceLookup().isControllerServiceEnabling(serviceId)) {
+ return new ValidationResult.Builder()
+ .input(context.getControllerServiceLookup().getControllerServiceName(serviceId))
+ .subject(getName())
+ .valid(false)
+ .explanation("Controller Service " + controllerService + " is disabled")
+ .build();
+ }
+
+ final Collection<ValidationResult> validationResults = controllerService.validate(context.getControllerServiceValidationContext(controllerService));
+ for (final ValidationResult result : validationResults) {
+ if (!result.isValid()) {
+ return new ValidationResult.Builder()
+ .input(input)
+ .subject(getName())
+ .valid(false)
+ .explanation("Controller Service is not valid: "
+ + ((result.getExplanation() == null || result.getExplanation().trim().isEmpty()) ? "(Service does not provide any explanation)" : result.getExplanation()))
+ .build();
+ }
+ }
+
+ return new ValidationResult.Builder()
+ .input(input)
+ .subject(getName())
+ .valid(true)
+ .build();
+ } else {
+ return new ValidationResult.Builder()
+ .input(input)
+ .subject(getName())
+ .valid(false)
+ .explanation("Invalid Controller Service: " + input + " is not a valid Controller Service Identifier or does not reference the correct type of Controller Service")
+ .build();
+ }
+ }
+
+ for (final Validator validator : validators) {
+ lastResult = validator.validate(this.name, input, context);
+ if (!lastResult.isValid()) {
+ break;
+ }
+ }
+ return lastResult;
+ }
+
+ public static final class Builder {
+
+ private String displayName = null;
+ private String name = null;
+ private String description = "";
+ private String defaultValue = null;
+ private List<AllowableValue> allowableValues = null;
+ private boolean required = false;
+ private boolean sensitive = false;
+ private boolean expressionLanguageSupported = false;
+ private boolean dynamic = false;
+ private Class<? extends ControllerService> controllerServiceDefinition;
+ private List<Validator> validators = new ArrayList<>();
+
+ public Builder fromPropertyDescriptor(final PropertyDescriptor specDescriptor) {
+ this.name = specDescriptor.name;
+ this.displayName = specDescriptor.displayName;
+ this.description = specDescriptor.description;
+ this.defaultValue = specDescriptor.defaultValue;
+ this.allowableValues = specDescriptor.allowableValues == null ? null : new ArrayList<>(specDescriptor.allowableValues);
+ this.required = specDescriptor.required;
+ this.sensitive = specDescriptor.sensitive;
+ this.dynamic = specDescriptor.dynamic;
+ this.expressionLanguageSupported = specDescriptor.expressionLanguageSupported;
+ this.controllerServiceDefinition = specDescriptor.getControllerServiceDefinition();
+ this.validators = new ArrayList<>(specDescriptor.validators);
+ return this;
+ }
+
+ /**
+ * Sets a unique id for the property. This field is optional and if not
+ * specified the PropertyDescriptor's name will be used as the
+ * identifying attribute. However, by supplying an id, the
+ * PropertyDescriptor's name can be changed without causing problems.
+ * This is beneficial because it allows a User Interface to represent
+ * the name differently.
+ *
+ * @param displayName of the property
+ * @return the builder
+ */
+ public Builder displayName(final String displayName) {
+ if (null != displayName) {
+ this.displayName = displayName;
+ }
+
+ return this;
+ }
+
+ /**
+ * Sets the property name.
+ *
+ * @param name of the property
+ * @return the builder
+ */
+ public Builder name(final String name) {
+ if (null != name) {
+ this.name = name;
+ }
+ return this;
+ }
+
+ /**
+ * Sets the value indicating whether or not this Property will support
+ * the Attribute Expression Language.
+ *
+ * @param supported true if yes; false otherwise
+ * @return the builder
+ */
+ public Builder expressionLanguageSupported(final boolean supported) {
+ this.expressionLanguageSupported = supported;
+ return this;
+ }
+
+ /**
+ * @param description of the property
+ * @return the builder
+ */
+ public Builder description(final String description) {
+ if (null != description) {
+ this.description = description;
+ }
+ return this;
+ }
+
+ /**
+ * Specifies the initial value and the default value that will be used
+ * if the user does not specify a value. When {@link #build()} is
+ * called, if Allowable Values have been set (see
+ * {@link #allowableValues(AllowableValue...)}) and this value is not
+ * one of those Allowable Values and Exception will be thrown. If the
+ * Allowable Values have been set using the
+ * {@link #allowableValues(AllowableValue...)} method, the default value
+ * should be set to the "Value" of the {@link AllowableValue} object
+ * (see {@link AllowableValue#getValue()}).
+ *
+ * @param value default value
+ * @return the builder
+ */
+ public Builder defaultValue(final String value) {
+ if (null != value) {
+ this.defaultValue = value;
+ }
+ return this;
+ }
+
+ public Builder dynamic(final boolean dynamic) {
+ this.dynamic = dynamic;
+ return this;
+ }
+
+ /**
+ * @param values contrained set of values
+ * @return the builder
+ */
+ public Builder allowableValues(final Set<String> values) {
+ if (null != values) {
+ this.allowableValues = new ArrayList<>();
+
+ for (final String value : values) {
+ this.allowableValues.add(new AllowableValue(value, value));
+ }
+ }
+ return this;
+ }
+
+ public <E extends Enum<E>> Builder allowableValues(final E[] values) {
+ if (null != values) {
+ this.allowableValues = new ArrayList<>();
+ for (final E value : values) {
+ allowableValues.add(new AllowableValue(value.name(), value.name()));
+ }
+ }
+ return this;
+ }
+
+ /**
+ * @param values constrained set of values
+ * @return the builder
+ */
+ public Builder allowableValues(final String... values) {
+ if (null != values) {
+ this.allowableValues = new ArrayList<>();
+ for (final String value : values) {
+ allowableValues.add(new AllowableValue(value, value));
+ }
+ }
+ return this;
+ }
+
+ /**
+ * Sets the Allowable Values for this Property
+ *
+ * @param values contrained set of values
+ * @return the builder
+ */
+ public Builder allowableValues(final AllowableValue... values) {
+ if (null != values) {
+ this.allowableValues = Arrays.asList(values);
+ }
+ return this;
+ }
+
+ /**
+ * @param required true if yes; false otherwise
+ * @return the builder
+ */
+ public Builder required(final boolean required) {
+ this.required = required;
+ return this;
+ }
+
+ /**
+ * @param sensitive true if sensitive; false otherwise
+ * @return the builder
+ */
+ public Builder sensitive(final boolean sensitive) {
+ this.sensitive = sensitive;
+ return this;
+ }
+
+ /**
+ * @param validator for the property
+ * @return the builder
+ */
+ public Builder addValidator(final Validator validator) {
+ if (validator != null) {
+ validators.add(validator);
+ }
+ return this;
+ }
+
+ /**
+ * Specifies that this property provides the identifier of a Controller
+ * Service that implements the given interface
+ *
+ * @param controllerServiceDefinition the interface that is implemented
+ * by the Controller Service
+ * @return the builder
+ */
+ public Builder identifiesControllerService(final Class<? extends ControllerService> controllerServiceDefinition) {
+ if (controllerServiceDefinition != null) {
+ this.controllerServiceDefinition = controllerServiceDefinition;
+ }
+ return this;
+ }
+
+ private boolean isValueAllowed(final String value) {
+ if (allowableValues == null || value == null) {
+ return true;
+ }
+
+ for (final AllowableValue allowableValue : allowableValues) {
+ if (allowableValue.getValue().equals(value)) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ /**
+ * @return a PropertyDescriptor as configured
+ *
+ * @throws IllegalStateException if allowable values are configured but
+ * no default value is set, or the default value is not contained within
+ * the allowable values.
+ */
+ public PropertyDescriptor build() {
+ if (name == null) {
+ throw new IllegalStateException("Must specify a name");
+ }
+ if (!isValueAllowed(defaultValue)) {
+ throw new IllegalStateException("Default value [" + defaultValue + "] is not in the set of allowable values");
+ }
+
+ return new PropertyDescriptor(this);
+ }
+ }
+
+ public String getDisplayName() {
+ return displayName;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ public String getDefaultValue() {
+ return defaultValue;
+ }
+
+ public boolean isRequired() {
+ return required;
+ }
+
+ public boolean isSensitive() {
+ return sensitive;
+ }
+
+ public boolean isDynamic() {
+ return dynamic;
+ }
+
+ public boolean isExpressionLanguageSupported() {
+ return expressionLanguageSupported;
+ }
+
+ public Class<? extends ControllerService> getControllerServiceDefinition() {
+ return controllerServiceDefinition;
+ }
+
+ public List<Validator> getValidators() {
+ return Collections.unmodifiableList(validators);
+ }
+
+ public List<AllowableValue> getAllowableValues() {
+ return allowableValues == null ? null : Collections.unmodifiableList(allowableValues);
+ }
+
+ @Override
+ public boolean equals(final Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (!(other instanceof PropertyDescriptor)) {
+ return false;
+ }
+ if (this == other) {
+ return true;
+ }
+
+ PropertyDescriptor desc = (PropertyDescriptor) other;
+ return this.name.equals(desc.name);
+ }
+
+ @Override
+ public int hashCode() {
+ return 287 + this.name.hashCode() * 47;
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + "[" + displayName + "]";
+ }
+
+ private static final class ConstrainedSetValidator implements Validator {
+
+ private static final String POSITIVE_EXPLANATION = "Given value found in allowed set";
+ private static final String NEGATIVE_EXPLANATION = "Given value not found in allowed set '%1$s'";
+ private static final String VALUE_DEMARCATOR = ", ";
+ private final String validStrings;
+ private final Collection<String> validValues;
+
+ /**
+ * Constructs a validator that will check if the given value is in the
+ * given set.
+ *
+ * @param validValues values which are acceptible
+ * @throws NullPointerException if the given validValues is null
+ */
+ private ConstrainedSetValidator(final Collection<AllowableValue> validValues) {
+ String validVals = "";
+ if (!validValues.isEmpty()) {
+ final StringBuilder valuesBuilder = new StringBuilder();
+ for (final AllowableValue value : validValues) {
+ valuesBuilder.append(value).append(VALUE_DEMARCATOR);
+ }
+ validVals = valuesBuilder.substring(0, valuesBuilder.length() - VALUE_DEMARCATOR.length());
+ }
+ validStrings = validVals;
+
+ this.validValues = new ArrayList<>(validValues.size());
+ for (final AllowableValue value : validValues) {
+ this.validValues.add(value.getValue());
+ }
+ }
+
+ @Override
+ public ValidationResult validate(final String subject, final String input, final ValidationContext context) {
+ final ValidationResult.Builder builder = new ValidationResult.Builder();
+ builder.input(input);
+ builder.subject(subject);
+ if (validValues.contains(input)) {
+ builder.valid(true);
+ builder.explanation(POSITIVE_EXPLANATION);
+ } else {
+ builder.valid(false);
+ builder.explanation(String.format(NEGATIVE_EXPLANATION, validStrings));
+ }
+ return builder.build();
+ }
+ }
+}
[11/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java b/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java
new file mode 100644
index 0000000..4ad3138
--- /dev/null
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java
@@ -0,0 +1,1037 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.OpenOption;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.regex.Pattern;
+
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.processor.FlowFileFilter;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Processor;
+import org.apache.nifi.processor.QueueSize;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.FlowFileAccessException;
+import org.apache.nifi.processor.exception.FlowFileHandlingException;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.io.InputStreamCallback;
+import org.apache.nifi.processor.io.OutputStreamCallback;
+import org.apache.nifi.processor.io.StreamCallback;
+import org.apache.nifi.provenance.ProvenanceReporter;
+import org.junit.Assert;
+
+public class MockProcessSession implements ProcessSession {
+
+ private final Map<Relationship, List<MockFlowFile>> transferMap = new ConcurrentHashMap<>();
+ private final MockFlowFileQueue processorQueue;
+ private final Set<Long> beingProcessed = new HashSet<>();
+
+ private final Map<Long, MockFlowFile> currentVersions = new HashMap<>();
+ private final Map<Long, MockFlowFile> originalVersions = new HashMap<>();
+ private final SharedSessionState sharedState;
+ private final Map<String, Long> counterMap = new HashMap<>();
+ private final MockProvenanceReporter provenanceReporter;
+
+ private boolean committed = false;
+ private boolean rolledback = false;
+ private int removedCount = 0;
+
+ public MockProcessSession(final SharedSessionState sharedState, final Processor processor) {
+ this.sharedState = sharedState;
+ this.processorQueue = sharedState.getFlowFileQueue();
+ provenanceReporter = new MockProvenanceReporter(this, sharedState, processor.getIdentifier(), processor.getClass().getSimpleName());
+ }
+
+ @Override
+ public void adjustCounter(final String name, final long delta, final boolean immediate) {
+ if (immediate) {
+ sharedState.adjustCounter(name, delta);
+ return;
+ }
+
+ Long counter = counterMap.get(name);
+ if (counter == null) {
+ counter = delta;
+ counterMap.put(name, counter);
+ return;
+ }
+
+ counter = counter + delta;
+ counterMap.put(name, counter);
+ }
+
+ @Override
+ public MockFlowFile clone(final FlowFile flowFile) {
+ validateState(flowFile);
+ final MockFlowFile newFlowFile = new MockFlowFile(sharedState.nextFlowFileId(), flowFile);
+ currentVersions.put(newFlowFile.getId(), newFlowFile);
+ beingProcessed.add(newFlowFile.getId());
+ return newFlowFile;
+ }
+
+ @Override
+ public MockFlowFile clone(final FlowFile flowFile, final long offset, final long size) {
+ validateState(flowFile);
+ if (offset + size > flowFile.getSize()) {
+ throw new FlowFileHandlingException("Specified offset of " + offset + " and size " + size + " exceeds size of " + flowFile.toString());
+ }
+
+ final MockFlowFile newFlowFile = new MockFlowFile(sharedState.nextFlowFileId(), flowFile);
+ final byte[] newContent = Arrays.copyOfRange(((MockFlowFile) flowFile).getData(), (int) offset, (int) (offset + size));
+ newFlowFile.setData(newContent);
+
+ currentVersions.put(newFlowFile.getId(), newFlowFile);
+ beingProcessed.add(newFlowFile.getId());
+ return newFlowFile;
+ }
+
+ @Override
+ public void commit() {
+ if (!beingProcessed.isEmpty()) {
+ throw new FlowFileHandlingException("Cannot commit session because the following FlowFiles have not been removed or transferred: " + beingProcessed);
+ }
+ committed = true;
+ beingProcessed.clear();
+ currentVersions.clear();
+ originalVersions.clear();
+
+ for (final Map.Entry<String, Long> entry : counterMap.entrySet()) {
+ sharedState.adjustCounter(entry.getKey(), entry.getValue());
+ }
+
+ sharedState.addProvenanceEvents(provenanceReporter.getEvents());
+ counterMap.clear();
+ }
+
+ /**
+ * Clear the 'committed' flag so that we can test that the next iteration of
+ * {@link nifi.processor.Processor#onTrigger} commits or rolls back the
+ * session
+ */
+ public void clearCommited() {
+ committed = false;
+ }
+
+ /**
+ * Clear the 'rolledBack' flag so that we can test that the next iteration
+ * of {@link nifi.processor.Processor#onTrigger} commits or rolls back the
+ * session
+ */
+ public void clearRollback() {
+ rolledback = false;
+ }
+
+ @Override
+ public MockFlowFile create() {
+ final MockFlowFile flowFile = new MockFlowFile(sharedState.nextFlowFileId());
+ currentVersions.put(flowFile.getId(), flowFile);
+ beingProcessed.add(flowFile.getId());
+ return flowFile;
+ }
+
+ @Override
+ public MockFlowFile create(final FlowFile flowFile) {
+ MockFlowFile newFlowFile = create();
+ newFlowFile = (MockFlowFile) inheritAttributes(flowFile, newFlowFile);
+ currentVersions.put(newFlowFile.getId(), newFlowFile);
+ beingProcessed.add(newFlowFile.getId());
+ return newFlowFile;
+ }
+
+ @Override
+ public MockFlowFile create(final Collection<FlowFile> flowFiles) {
+ MockFlowFile newFlowFile = create();
+ newFlowFile = (MockFlowFile) inheritAttributes(flowFiles, newFlowFile);
+ currentVersions.put(newFlowFile.getId(), newFlowFile);
+ beingProcessed.add(newFlowFile.getId());
+ return newFlowFile;
+ }
+
+ @Override
+ public void exportTo(final FlowFile flowFile, final OutputStream out) {
+ validateState(flowFile);
+ if (flowFile == null || out == null) {
+ throw new IllegalArgumentException("arguments cannot be null");
+ }
+
+ if (!(flowFile instanceof MockFlowFile)) {
+ throw new IllegalArgumentException("Cannot export a flow file that I did not create");
+ }
+
+ final MockFlowFile mock = (MockFlowFile) flowFile;
+
+ try {
+ out.write(mock.getData());
+ } catch (final IOException e) {
+ throw new FlowFileAccessException(e.toString(), e);
+ }
+ }
+
+ @Override
+ public void exportTo(final FlowFile flowFile, final Path path, final boolean append) {
+ validateState(flowFile);
+ if (flowFile == null || path == null) {
+ throw new IllegalArgumentException("argument cannot be null");
+ }
+ if (!(flowFile instanceof MockFlowFile)) {
+ throw new IllegalArgumentException("Cannot export a flow file that I did not create");
+ }
+
+ final MockFlowFile mock = (MockFlowFile) flowFile;
+
+ final OpenOption mode = append ? StandardOpenOption.APPEND : StandardOpenOption.CREATE;
+
+ try (final OutputStream out = Files.newOutputStream(path, mode)) {
+ out.write(mock.getData());
+ } catch (final IOException e) {
+ throw new FlowFileAccessException(e.toString(), e);
+ }
+ }
+
+ @Override
+ public MockFlowFile get() {
+ final MockFlowFile flowFile = processorQueue.poll();
+ if (flowFile != null) {
+ beingProcessed.add(flowFile.getId());
+ currentVersions.put(flowFile.getId(), flowFile);
+ originalVersions.put(flowFile.getId(), flowFile);
+ }
+ return flowFile;
+ }
+
+ @Override
+ public List<FlowFile> get(final int maxResults) {
+ final List<FlowFile> flowFiles = new ArrayList<>(Math.min(500, maxResults));
+ for (int i = 0; i < maxResults; i++) {
+ final MockFlowFile nextFlowFile = get();
+ if (nextFlowFile == null) {
+ return flowFiles;
+ }
+
+ flowFiles.add(nextFlowFile);
+ }
+
+ return flowFiles;
+ }
+
+ @Override
+ public List<FlowFile> get(final FlowFileFilter filter) {
+ final List<FlowFile> flowFiles = new ArrayList<>();
+ final List<MockFlowFile> unselected = new ArrayList<>();
+
+ while (true) {
+ final MockFlowFile flowFile = processorQueue.poll();
+ if (flowFile == null) {
+ break;
+ }
+
+ final FlowFileFilter.FlowFileFilterResult filterResult = filter.filter(flowFile);
+ if (filterResult.isAccept()) {
+ flowFiles.add(flowFile);
+
+ beingProcessed.add(flowFile.getId());
+ currentVersions.put(flowFile.getId(), flowFile);
+ originalVersions.put(flowFile.getId(), flowFile);
+ } else {
+ unselected.add(flowFile);
+ }
+
+ if (!filterResult.isContinue()) {
+ break;
+ }
+ }
+
+ processorQueue.addAll(unselected);
+ return flowFiles;
+ }
+
+ @Override
+ public QueueSize getQueueSize() {
+ return processorQueue.size();
+ }
+
+ @Override
+ public MockFlowFile importFrom(final InputStream in, final FlowFile flowFile) {
+ validateState(flowFile);
+ if (in == null || flowFile == null) {
+ throw new IllegalArgumentException("argument cannot be null");
+ }
+ if (!(flowFile instanceof MockFlowFile)) {
+ throw new IllegalArgumentException("Cannot export a flow file that I did not create");
+ }
+ final MockFlowFile mock = (MockFlowFile) flowFile;
+
+ final MockFlowFile newFlowFile = new MockFlowFile(mock.getId(), flowFile);
+ currentVersions.put(newFlowFile.getId(), newFlowFile);
+ try {
+ final byte[] data = readFully(in);
+ newFlowFile.setData(data);
+ return newFlowFile;
+ } catch (final IOException e) {
+ throw new FlowFileAccessException(e.toString(), e);
+ }
+ }
+
+ @Override
+ public MockFlowFile importFrom(final Path path, final boolean keepSourceFile, final FlowFile flowFile) {
+ validateState(flowFile);
+ if (path == null || flowFile == null) {
+ throw new IllegalArgumentException("argument cannot be null");
+ }
+ if (!(flowFile instanceof MockFlowFile)) {
+ throw new IllegalArgumentException("Cannot export a flow file that I did not create");
+ }
+ final MockFlowFile mock = (MockFlowFile) flowFile;
+ MockFlowFile newFlowFile = new MockFlowFile(mock.getId(), flowFile);
+ currentVersions.put(newFlowFile.getId(), newFlowFile);
+
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ try {
+ Files.copy(path, baos);
+ } catch (final IOException e) {
+ throw new FlowFileAccessException(e.toString(), e);
+ }
+
+ newFlowFile.setData(baos.toByteArray());
+ newFlowFile = putAttribute(newFlowFile, CoreAttributes.FILENAME.key(), path.getFileName().toString());
+ return newFlowFile;
+ }
+
+ @Override
+ public MockFlowFile merge(final Collection<FlowFile> sources, final FlowFile destination) {
+ for (final FlowFile source : sources) {
+ validateState(source);
+ }
+ validateState(destination);
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ for (final FlowFile flowFile : sources) {
+ final MockFlowFile mock = (MockFlowFile) flowFile;
+ final byte[] data = mock.getData();
+ try {
+ baos.write(data);
+ } catch (final IOException e) {
+ throw new AssertionError("Failed to write to BAOS");
+ }
+ }
+
+ final MockFlowFile newFlowFile = new MockFlowFile(destination.getId(), destination);
+ newFlowFile.setData(baos.toByteArray());
+ currentVersions.put(newFlowFile.getId(), newFlowFile);
+
+ return newFlowFile;
+ }
+
+ @Override
+ public MockFlowFile putAllAttributes(final FlowFile flowFile, final Map<String, String> attrs) {
+ validateState(flowFile);
+ if (attrs == null || flowFile == null) {
+ throw new IllegalArgumentException("argument cannot be null");
+ }
+ if (!(flowFile instanceof MockFlowFile)) {
+ throw new IllegalArgumentException("Cannot update attributes of a flow file that I did not create");
+ }
+ final MockFlowFile mock = (MockFlowFile) flowFile;
+ final MockFlowFile newFlowFile = new MockFlowFile(mock.getId(), flowFile);
+ currentVersions.put(newFlowFile.getId(), newFlowFile);
+
+ newFlowFile.putAttributes(attrs);
+ return newFlowFile;
+ }
+
+ @Override
+ public MockFlowFile putAttribute(final FlowFile flowFile, final String attrName, final String attrValue) {
+ validateState(flowFile);
+ if (attrName == null || attrValue == null || flowFile == null) {
+ throw new IllegalArgumentException("argument cannot be null");
+ }
+ if (!(flowFile instanceof MockFlowFile)) {
+ throw new IllegalArgumentException("Cannot update attributes of a flow file that I did not create");
+ }
+
+ if ("uuid".equals(attrName)) {
+ Assert.fail("Should not be attempting to set FlowFile UUID via putAttribute. This will be ignored in production");
+ }
+
+ final MockFlowFile mock = (MockFlowFile) flowFile;
+ final MockFlowFile newFlowFile = new MockFlowFile(mock.getId(), flowFile);
+ currentVersions.put(newFlowFile.getId(), newFlowFile);
+
+ final Map<String, String> attrs = new HashMap<>();
+ attrs.put(attrName, attrValue);
+ newFlowFile.putAttributes(attrs);
+ return newFlowFile;
+ }
+
+ @Override
+ public void read(final FlowFile flowFile, final InputStreamCallback callback) {
+ if (callback == null || flowFile == null) {
+ throw new IllegalArgumentException("argument cannot be null");
+ }
+
+ validateState(flowFile);
+ if (!(flowFile instanceof MockFlowFile)) {
+ throw new IllegalArgumentException("Cannot export a flow file that I did not create");
+ }
+ final MockFlowFile mock = (MockFlowFile) flowFile;
+
+ final ByteArrayInputStream bais = new ByteArrayInputStream(mock.getData());
+ try {
+ callback.process(bais);
+ } catch (final IOException e) {
+ throw new ProcessException(e.toString(), e);
+ }
+ }
+
+ @Override
+ public void remove(final FlowFile flowFile) {
+ validateState(flowFile);
+ final Iterator<Long> itr = beingProcessed.iterator();
+ while (itr.hasNext()) {
+ final Long ffId = itr.next();
+ if (ffId != null && ffId.equals(flowFile.getId())) {
+ itr.remove();
+ beingProcessed.remove(ffId);
+ removedCount++;
+ currentVersions.remove(ffId);
+ return;
+ }
+ }
+
+ throw new ProcessException(flowFile + " not found in queue");
+ }
+
+ @Override
+ public void remove(final Collection<FlowFile> flowFiles) {
+ for (final FlowFile flowFile : flowFiles) {
+ validateState(flowFile);
+ }
+
+ for (final FlowFile flowFile : flowFiles) {
+ remove(flowFile);
+ }
+ }
+
+ @Override
+ public MockFlowFile removeAllAttributes(final FlowFile flowFile, final Set<String> attrNames) {
+ validateState(flowFile);
+ if (attrNames == null || flowFile == null) {
+ throw new IllegalArgumentException("argument cannot be null");
+ }
+ if (!(flowFile instanceof MockFlowFile)) {
+ throw new IllegalArgumentException("Cannot export a flow file that I did not create");
+ }
+ final MockFlowFile mock = (MockFlowFile) flowFile;
+
+ final MockFlowFile newFlowFile = new MockFlowFile(mock.getId(), flowFile);
+ currentVersions.put(newFlowFile.getId(), newFlowFile);
+
+ newFlowFile.removeAttributes(attrNames);
+ return newFlowFile;
+ }
+
+ @Override
+ public MockFlowFile removeAllAttributes(final FlowFile flowFile, final Pattern keyPattern) {
+ validateState(flowFile);
+ if (flowFile == null) {
+ throw new IllegalArgumentException("flowFile cannot be null");
+ }
+ if (keyPattern == null) {
+ return (MockFlowFile) flowFile;
+ }
+
+ final Set<String> attrsToRemove = new HashSet<>();
+ for (final String key : flowFile.getAttributes().keySet()) {
+ if (keyPattern.matcher(key).matches()) {
+ attrsToRemove.add(key);
+ }
+ }
+
+ return removeAllAttributes(flowFile, attrsToRemove);
+ }
+
+ @Override
+ public MockFlowFile removeAttribute(final FlowFile flowFile, final String attrName) {
+ validateState(flowFile);
+ if (attrName == null || flowFile == null) {
+ throw new IllegalArgumentException("argument cannot be null");
+ }
+ if (!(flowFile instanceof MockFlowFile)) {
+ throw new IllegalArgumentException("Cannot export a flow file that I did not create");
+ }
+ final MockFlowFile mock = (MockFlowFile) flowFile;
+ final MockFlowFile newFlowFile = new MockFlowFile(mock.getId(), flowFile);
+ currentVersions.put(newFlowFile.getId(), newFlowFile);
+
+ final Set<String> attrNames = new HashSet<>();
+ attrNames.add(attrName);
+ newFlowFile.removeAttributes(attrNames);
+ return newFlowFile;
+ }
+
+ @Override
+ public void rollback() {
+ rollback(false);
+ }
+
+ @Override
+ public void rollback(final boolean penalize) {
+ for (final List<MockFlowFile> list : transferMap.values()) {
+ for (final MockFlowFile flowFile : list) {
+ processorQueue.offer(flowFile);
+ }
+ }
+
+ for (final Long flowFileId : beingProcessed) {
+ final MockFlowFile flowFile = originalVersions.get(flowFileId);
+ if (flowFile != null) {
+ processorQueue.offer(flowFile);
+ }
+ }
+
+ rolledback = true;
+ beingProcessed.clear();
+ currentVersions.clear();
+ originalVersions.clear();
+ transferMap.clear();
+ clearTransferState();
+ }
+
+ @Override
+ public void transfer(final FlowFile flowFile) {
+ validateState(flowFile);
+ if (!(flowFile instanceof MockFlowFile)) {
+ throw new IllegalArgumentException("I only accept MockFlowFile");
+ }
+
+ beingProcessed.remove(flowFile.getId());
+ processorQueue.offer((MockFlowFile) flowFile);
+ }
+
+ @Override
+ public void transfer(final Collection<FlowFile> flowFiles) {
+ for (final FlowFile flowFile : flowFiles) {
+ transfer(flowFile);
+ }
+ }
+
+ @Override
+ public void transfer(final FlowFile flowFile, final Relationship relationship) {
+ if (relationship == Relationship.SELF) {
+ transfer(flowFile);
+ return;
+ }
+
+ validateState(flowFile);
+ List<MockFlowFile> list = transferMap.get(relationship);
+ if (list == null) {
+ list = new ArrayList<>();
+ transferMap.put(relationship, list);
+ }
+
+ beingProcessed.remove(flowFile.getId());
+ list.add((MockFlowFile) flowFile);
+ }
+
+ @Override
+ public void transfer(final Collection<FlowFile> flowFiles, final Relationship relationship) {
+ if (relationship == Relationship.SELF) {
+ transfer(flowFiles);
+ return;
+ }
+
+ for (final FlowFile flowFile : flowFiles) {
+ validateState(flowFile);
+ }
+
+ List<MockFlowFile> list = transferMap.get(relationship);
+ if (list == null) {
+ list = new ArrayList<>();
+ transferMap.put(relationship, list);
+ }
+
+ for (final FlowFile flowFile : flowFiles) {
+ beingProcessed.remove(flowFile.getId());
+ list.add((MockFlowFile) flowFile);
+ }
+ }
+
+ @Override
+ public MockFlowFile write(final FlowFile flowFile, final OutputStreamCallback callback) {
+ validateState(flowFile);
+ if (callback == null || flowFile == null) {
+ throw new IllegalArgumentException("argument cannot be null");
+ }
+ if (!(flowFile instanceof MockFlowFile)) {
+ throw new IllegalArgumentException("Cannot export a flow file that I did not create");
+ }
+ final MockFlowFile mock = (MockFlowFile) flowFile;
+
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ try {
+ callback.process(baos);
+ } catch (final IOException e) {
+ throw new ProcessException(e.toString(), e);
+ }
+
+ final MockFlowFile newFlowFile = new MockFlowFile(mock.getId(), flowFile);
+ currentVersions.put(newFlowFile.getId(), newFlowFile);
+
+ newFlowFile.setData(baos.toByteArray());
+ return newFlowFile;
+ }
+
+ @Override
+ public FlowFile append(final FlowFile flowFile, final OutputStreamCallback callback) {
+ validateState(flowFile);
+ if (callback == null || flowFile == null) {
+ throw new IllegalArgumentException("argument cannot be null");
+ }
+ if (!(flowFile instanceof MockFlowFile)) {
+ throw new IllegalArgumentException("Cannot export a flow file that I did not create");
+ }
+ final MockFlowFile mock = (MockFlowFile) flowFile;
+
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ try {
+ baos.write(mock.getData());
+ callback.process(baos);
+ } catch (final IOException e) {
+ throw new ProcessException(e.toString(), e);
+ }
+
+ final MockFlowFile newFlowFile = new MockFlowFile(mock.getId(), flowFile);
+ currentVersions.put(newFlowFile.getId(), newFlowFile);
+
+ newFlowFile.setData(baos.toByteArray());
+ return newFlowFile;
+ }
+
+ @Override
+ public MockFlowFile write(final FlowFile flowFile, final StreamCallback callback) {
+ validateState(flowFile);
+ if (callback == null || flowFile == null) {
+ throw new IllegalArgumentException("argument cannot be null");
+ }
+ if (!(flowFile instanceof MockFlowFile)) {
+ throw new IllegalArgumentException("Cannot export a flow file that I did not create");
+ }
+ final MockFlowFile mock = (MockFlowFile) flowFile;
+
+ final ByteArrayInputStream in = new ByteArrayInputStream(mock.getData());
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try {
+ callback.process(in, out);
+ } catch (final IOException e) {
+ throw new ProcessException(e.toString(), e);
+ }
+
+ final MockFlowFile newFlowFile = new MockFlowFile(mock.getId(), flowFile);
+ currentVersions.put(newFlowFile.getId(), newFlowFile);
+ newFlowFile.setData(out.toByteArray());
+
+ return newFlowFile;
+ }
+
+ private byte[] readFully(final InputStream in) throws IOException {
+ final byte[] buffer = new byte[4096];
+ int len;
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ while ((len = in.read(buffer)) >= 0) {
+ baos.write(buffer, 0, len);
+ }
+
+ return baos.toByteArray();
+ }
+
+ public List<MockFlowFile> getFlowFilesForRelationship(final Relationship relationship) {
+ List<MockFlowFile> list = this.transferMap.get(relationship);
+ if (list == null) {
+ list = new ArrayList<>();
+ }
+
+ return list;
+ }
+
+ /**
+ * @param relationship to get flowfiles for
+ * @return a List of FlowFiles in the order in which they were transferred
+ * to the given relationship
+ */
+ public List<MockFlowFile> getFlowFilesForRelationship(final String relationship) {
+ final Relationship procRel = new Relationship.Builder().name(relationship).build();
+ return getFlowFilesForRelationship(procRel);
+ }
+
+ public MockFlowFile createFlowFile(final File file) throws IOException {
+ return createFlowFile(Files.readAllBytes(file.toPath()));
+ }
+
+ public MockFlowFile createFlowFile(final byte[] data) {
+ final MockFlowFile flowFile = create();
+ flowFile.setData(data);
+ return flowFile;
+ }
+
+ public MockFlowFile createFlowFile(final byte[] data, final Map<String, String> attrs) {
+ final MockFlowFile ff = createFlowFile(data);
+ ff.putAttributes(attrs);
+ return ff;
+ }
+
+ @Override
+ public MockFlowFile merge(Collection<FlowFile> sources, FlowFile destination, byte[] header, byte[] footer, byte[] demarcator) {
+ for (final FlowFile flowFile : sources) {
+ validateState(flowFile);
+ }
+ validateState(destination);
+
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ try {
+ if (header != null) {
+ baos.write(header);
+ }
+
+ int count = 0;
+ for (final FlowFile flowFile : sources) {
+ baos.write(((MockFlowFile) flowFile).getData());
+ if (demarcator != null && ++count != sources.size()) {
+ baos.write(demarcator);
+ }
+ }
+
+ if (footer != null) {
+ baos.write(footer);
+ }
+ } catch (final IOException e) {
+ throw new AssertionError("failed to write data to BAOS");
+ }
+
+ final MockFlowFile newFlowFile = new MockFlowFile(destination.getId(), destination);
+ newFlowFile.setData(baos.toByteArray());
+ currentVersions.put(newFlowFile.getId(), newFlowFile);
+
+ return newFlowFile;
+ }
+
+ private void validateState(final FlowFile flowFile) {
+ Objects.requireNonNull(flowFile);
+ final FlowFile currentVersion = currentVersions.get(flowFile.getId());
+ if (currentVersion == null) {
+ throw new FlowFileHandlingException(flowFile + " is not known in this session");
+ }
+
+ if (currentVersion != flowFile) {
+ throw new FlowFileHandlingException(flowFile + " is not the most recent version of this flow file within this session");
+ }
+
+ for (final List<MockFlowFile> flowFiles : transferMap.values()) {
+ if (flowFiles.contains(flowFile)) {
+ throw new IllegalStateException(flowFile + " has already been transferred");
+ }
+ }
+ }
+
+ /**
+ * Inherits the attributes from the given source flow file into another flow
+ * file. The UUID of the source becomes the parent UUID of this flow file.
+ * If a parent uuid had previously been established it will be replaced by
+ * the uuid of the given source
+ *
+ * @param source the FlowFile from which to copy attributes
+ * @param destination the FlowFile to which to copy attributes
+ */
+ private FlowFile inheritAttributes(final FlowFile source, final FlowFile destination) {
+ if (source == null || destination == null || source == destination) {
+ return destination; //don't need to inherit from ourselves
+ }
+ final FlowFile updated = putAllAttributes(destination, source.getAttributes());
+ getProvenanceReporter().fork(source, Collections.singletonList(updated));
+ return updated;
+ }
+
+ /**
+ * Inherits the attributes from the given source flow files into the
+ * destination flow file. The UUIDs of the sources becomes the parent UUIDs
+ * of the destination flow file. Only attributes which is common to all
+ * source items is copied into this flow files attributes. Any previously
+ * established parent UUIDs will be replaced by the UUIDs of the sources. It
+ * will capture the uuid of a certain number of source objects and may not
+ * capture all of them. How many it will capture is unspecified.
+ *
+ * @param sources to inherit common attributes from
+ */
+ private FlowFile inheritAttributes(final Collection<FlowFile> sources, final FlowFile destination) {
+ final StringBuilder parentUuidBuilder = new StringBuilder();
+ int uuidsCaptured = 0;
+ for (final FlowFile source : sources) {
+ if (source == destination) {
+ continue; //don't want to capture parent uuid of this. Something can't be a child of itself
+ }
+ final String sourceUuid = source.getAttribute(CoreAttributes.UUID.key());
+ if (sourceUuid != null && !sourceUuid.trim().isEmpty()) {
+ uuidsCaptured++;
+ if (parentUuidBuilder.length() > 0) {
+ parentUuidBuilder.append(",");
+ }
+ parentUuidBuilder.append(sourceUuid);
+ }
+
+ if (uuidsCaptured > 100) {
+ break;
+ }
+ }
+
+ final FlowFile updated = putAllAttributes(destination, intersectAttributes(sources));
+ getProvenanceReporter().join(sources, updated);
+ return updated;
+ }
+
+ /**
+ * Returns the attributes that are common to every flow file given. The key
+ * and value must match exactly.
+ *
+ * @param flowFileList a list of flow files
+ *
+ * @return the common attributes
+ */
+ private static Map<String, String> intersectAttributes(final Collection<FlowFile> flowFileList) {
+ final Map<String, String> result = new HashMap<>();
+ //trivial cases
+ if (flowFileList == null || flowFileList.isEmpty()) {
+ return result;
+ } else if (flowFileList.size() == 1) {
+ result.putAll(flowFileList.iterator().next().getAttributes());
+ }
+
+ /*
+ * Start with the first attribute map and only put an entry to the
+ * resultant map if it is common to every map.
+ */
+ final Map<String, String> firstMap = flowFileList.iterator().next().getAttributes();
+
+ outer:
+ for (final Map.Entry<String, String> mapEntry : firstMap.entrySet()) {
+ final String key = mapEntry.getKey();
+ final String value = mapEntry.getValue();
+ for (final FlowFile flowFile : flowFileList) {
+ final Map<String, String> currMap = flowFile.getAttributes();
+ final String curVal = currMap.get(key);
+ if (curVal == null || !curVal.equals(value)) {
+ continue outer;
+ }
+ }
+ result.put(key, value);
+ }
+
+ return result;
+ }
+
+ /**
+ * Assert that {@link #commit()} has been called
+ */
+ public void assertCommitted() {
+ Assert.assertTrue("Session was not committed", committed);
+ }
+
+ /**
+ * Assert that {@link #commit()} has not been called
+ */
+ public void assertNotCommitted() {
+ Assert.assertFalse("Session was committed", committed);
+ }
+
+ /**
+ * Assert that {@link #rollback()} has been called
+ */
+ public void assertRolledBack() {
+ Assert.assertTrue("Session was not rolled back", rolledback);
+ }
+
+ /**
+ * Assert that {@link #rollback()} has not been called
+ */
+ public void assertNotRolledBack() {
+ Assert.assertFalse("Session was rolled back", rolledback);
+ }
+
+ /**
+ * Assert that the number of FlowFiles transferred to the given relationship
+ * is equal to the given count
+ *
+ * @param relationship to validate transfer count of
+ * @param count items transfer to given relationship
+ */
+ public void assertTransferCount(final Relationship relationship, final int count) {
+ final int transferCount = getFlowFilesForRelationship(relationship).size();
+ Assert.assertEquals("Expected " + count + " FlowFiles to be transferred to "
+ + relationship + " but actual transfer count was " + transferCount, count, transferCount);
+ }
+
+ /**
+ * Assert that the number of FlowFiles transferred to the given relationship
+ * is equal to the given count
+ *
+ * @param relationship to validate transfer count of
+ * @param count items transfer to given relationship
+ */
+ public void assertTransferCount(final String relationship, final int count) {
+ assertTransferCount(new Relationship.Builder().name(relationship).build(), count);
+ }
+
+ /**
+ * Assert that there are no FlowFiles left on the input queue.
+ */
+ public void assertQueueEmpty() {
+ Assert.assertTrue("FlowFile Queue has " + this.processorQueue.size() + " items", this.processorQueue.isEmpty());
+ }
+
+ /**
+ * Assert that at least one FlowFile is on the input queue
+ */
+ public void assertQueueNotEmpty() {
+ Assert.assertFalse("FlowFile Queue is empty", this.processorQueue.isEmpty());
+ }
+
+ /**
+ * Asserts that all FlowFiles that were transferred were transferred to the
+ * given relationship
+ *
+ * @param relationship to check for transferred flow files
+ */
+ public void assertAllFlowFilesTransferred(final String relationship) {
+ assertAllFlowFilesTransferred(new Relationship.Builder().name(relationship).build());
+ }
+
+ /**
+ * Asserts that all FlowFiles that were transferred were transferred to the
+ * given relationship
+ *
+ * @param relationship to validate
+ */
+ public void assertAllFlowFilesTransferred(final Relationship relationship) {
+ for (final Map.Entry<Relationship, List<MockFlowFile>> entry : transferMap.entrySet()) {
+ final Relationship rel = entry.getKey();
+ final List<MockFlowFile> flowFiles = entry.getValue();
+
+ if (!rel.equals(relationship) && flowFiles != null && !flowFiles.isEmpty()) {
+ Assert.fail("Expected all Transferred FlowFiles to go to " + relationship + " but " + flowFiles.size() + " were routed to " + rel);
+ }
+ }
+ }
+
+ /**
+ * Removes all state information about FlowFiles that have been transferred
+ */
+ public void clearTransferState() {
+ this.transferMap.clear();
+ }
+
+ /**
+ * Asserts that all FlowFiles that were transferred were transferred to the
+ * given relationship and that the number of FlowFiles transferred is equal
+ * to <code>count</code>
+ *
+ * @param relationship to validate
+ * @param count number of items sent to that relationship (expected)
+ */
+ public void assertAllFlowFilesTransferred(final Relationship relationship, final int count) {
+ assertAllFlowFilesTransferred(relationship);
+ assertTransferCount(relationship, count);
+ }
+
+ /**
+ * Asserts that all FlowFiles that were transferred were transferred to the
+ * given relationship and that the number of FlowFiles transferred is equal
+ * to <code>count</code>
+ *
+ * @param relationship to validate
+ * @param count number of items sent to that relationship (expected)
+ */
+ public void assertAllFlowFilesTransferred(final String relationship, final int count) {
+ assertAllFlowFilesTransferred(new Relationship.Builder().name(relationship).build(), count);
+ }
+
+ /**
+ * @return the number of FlowFiles that were removed
+ */
+ public int getRemovedCount() {
+ return removedCount;
+ }
+
+ @Override
+ public ProvenanceReporter getProvenanceReporter() {
+ return provenanceReporter;
+ }
+
+ @Override
+ public MockFlowFile penalize(final FlowFile flowFile) {
+ validateState(flowFile);
+ final MockFlowFile mockFlowFile = (MockFlowFile) flowFile;
+ mockFlowFile.setPenalized();
+ return mockFlowFile;
+ }
+
+ public byte[] getContentAsByteArray(final MockFlowFile flowFile) {
+ validateState(flowFile);
+ return flowFile.getData();
+ }
+
+ /**
+ * Checks if a FlowFile is known in this session.
+ *
+ * @param flowFile
+ * the FlowFile to check
+ * @return <code>true</code> if the FlowFile is known in this session,
+ * <code>false</code> otherwise.
+ */
+ boolean isFlowFileKnown(final FlowFile flowFile) {
+ final FlowFile curFlowFile = currentVersions.get(flowFile.getId());
+ if (curFlowFile == null) {
+ return false;
+ }
+
+ final String curUuid = curFlowFile.getAttribute(CoreAttributes.UUID.key());
+ final String providedUuid = curFlowFile.getAttribute(CoreAttributes.UUID.key());
+ if (!curUuid.equals(providedUuid)) {
+ return false;
+ }
+
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessorInitializationContext.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessorInitializationContext.java b/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessorInitializationContext.java
new file mode 100644
index 0000000..2e5d3eb
--- /dev/null
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessorInitializationContext.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import java.util.Set;
+import java.util.UUID;
+
+import org.apache.nifi.controller.ControllerService;
+import org.apache.nifi.controller.ControllerServiceLookup;
+import org.apache.nifi.logging.ProcessorLog;
+import org.apache.nifi.processor.Processor;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+
+public class MockProcessorInitializationContext implements ProcessorInitializationContext, ControllerServiceLookup {
+
+ private final ProcessorLog logger;
+ private final String processorId;
+ private final MockProcessContext context;
+
+ public MockProcessorInitializationContext(final Processor processor, final MockProcessContext context) {
+ processorId = UUID.randomUUID().toString();
+ logger = new MockProcessorLog(processorId, processor);
+ this.context = context;
+ }
+
+ @Override
+ public String getIdentifier() {
+ return processorId;
+ }
+
+ @Override
+ public ProcessorLog getLogger() {
+ return logger;
+ }
+
+ @Override
+ public Set<String> getControllerServiceIdentifiers(final Class<? extends ControllerService> serviceType) {
+ return context.getControllerServiceIdentifiers(serviceType);
+ }
+
+ @Override
+ public ControllerService getControllerService(final String identifier) {
+ return context.getControllerService(identifier);
+ }
+
+ @Override
+ public ControllerServiceLookup getControllerServiceLookup() {
+ return this;
+ }
+
+ @Override
+ public String getControllerServiceName(String serviceIdentifier) {
+ return context.getControllerServiceName(serviceIdentifier);
+ }
+
+ @Override
+ public boolean isControllerServiceEnabled(String serviceIdentifier) {
+ return context.isControllerServiceEnabled(serviceIdentifier);
+ }
+
+ @Override
+ public boolean isControllerServiceEnabled(ControllerService service) {
+ return context.isControllerServiceEnabled(service);
+ }
+
+ @Override
+ public boolean isControllerServiceEnabling(String serviceIdentifier) {
+ return context.isControllerServiceEnabling(serviceIdentifier);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessorLog.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessorLog.java b/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessorLog.java
new file mode 100644
index 0000000..837784b
--- /dev/null
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessorLog.java
@@ -0,0 +1,276 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import org.apache.nifi.logging.ProcessorLog;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class MockProcessorLog implements ProcessorLog {
+
+ private final Logger logger;
+ private final Object component;
+
+ public MockProcessorLog(final String componentId, final Object component) {
+ this.logger = LoggerFactory.getLogger(component.getClass());
+ this.component = component;
+ }
+
+ private Object[] addProcessor(final Object[] originalArgs) {
+ return prependToArgs(originalArgs, component);
+ }
+
+ private Object[] addProcessorAndThrowable(final Object[] os, final Throwable t) {
+ final Object[] modifiedArgs = new Object[os.length + 2];
+ modifiedArgs[0] = component.toString();
+ for (int i = 0; i < os.length; i++) {
+ modifiedArgs[i + 1] = os[i];
+ }
+ modifiedArgs[modifiedArgs.length - 1] = t.toString();
+
+ return modifiedArgs;
+ }
+
+ private Object[] prependToArgs(final Object[] originalArgs, final Object... toAdd) {
+ final Object[] newArgs = new Object[originalArgs.length + toAdd.length];
+ System.arraycopy(toAdd, 0, newArgs, 0, toAdd.length);
+ System.arraycopy(originalArgs, 0, newArgs, toAdd.length, originalArgs.length);
+ return newArgs;
+ }
+
+ private Object[] translateException(final Object[] os) {
+ if (os != null && os.length > 0 && (os[os.length - 1] instanceof Throwable)) {
+ final Object[] osCopy = new Object[os.length];
+ osCopy[osCopy.length - 1] = os[os.length - 1].toString();
+ System.arraycopy(os, 0, osCopy, 0, os.length - 1);
+ return osCopy;
+ }
+ return os;
+ }
+
+ private boolean lastArgIsException(final Object[] os) {
+ return (os != null && os.length > 0 && (os[os.length - 1] instanceof Throwable));
+ }
+
+ @Override
+ public void warn(final String msg, final Throwable t) {
+ warn("{} " + msg, new Object[]{component}, t);
+ }
+
+ @Override
+ public void warn(String msg, Object[] os) {
+ if (lastArgIsException(os)) {
+ warn(msg, translateException(os), (Throwable) os[os.length - 1]);
+ } else {
+ msg = "{} " + msg;
+ os = addProcessor(os);
+ logger.warn(msg, os);
+ }
+ }
+
+ @Override
+ public void warn(String msg, Object[] os, final Throwable t) {
+ os = addProcessorAndThrowable(os, t);
+ msg = "{} " + msg + ": {}";
+
+ logger.warn(msg, os);
+ if (logger.isDebugEnabled()) {
+ logger.warn("", t);
+ }
+ }
+
+ @Override
+ public void warn(String msg) {
+ msg = "{} " + msg;
+ logger.warn(msg, component);
+ }
+
+ @Override
+ public void trace(String msg, Throwable t) {
+ msg = "{} " + msg;
+ final Object[] os = {component};
+ logger.trace(msg, os, t);
+ }
+
+ @Override
+ public void trace(String msg, Object[] os) {
+ msg = "{} " + msg;
+ os = addProcessor(os);
+ logger.trace(msg, os);
+ }
+
+ @Override
+ public void trace(String msg) {
+ msg = "{} " + msg;
+ final Object[] os = {component};
+ logger.trace(msg, os);
+ }
+
+ @Override
+ public void trace(String msg, Object[] os, Throwable t) {
+ os = addProcessorAndThrowable(os, t);
+ msg = "{} " + msg + ": {}";
+
+ logger.trace(msg, os);
+ logger.trace("", t);
+ }
+
+ @Override
+ public boolean isWarnEnabled() {
+ return logger.isWarnEnabled();
+ }
+
+ @Override
+ public boolean isTraceEnabled() {
+ return logger.isTraceEnabled();
+ }
+
+ @Override
+ public boolean isInfoEnabled() {
+ return logger.isInfoEnabled();
+ }
+
+ @Override
+ public boolean isErrorEnabled() {
+ return logger.isErrorEnabled();
+ }
+
+ @Override
+ public boolean isDebugEnabled() {
+ return logger.isDebugEnabled();
+ }
+
+ @Override
+ public void info(String msg, Throwable t) {
+ msg = "{} " + msg;
+ final Object[] os = {component};
+
+ logger.info(msg, os);
+ if (logger.isDebugEnabled()) {
+ logger.info("", t);
+ }
+ }
+
+ @Override
+ public void info(String msg, Object[] os) {
+ msg = "{} " + msg;
+ os = addProcessor(os);
+
+ logger.info(msg, os);
+ }
+
+ @Override
+ public void info(String msg) {
+ msg = "{} " + msg;
+ final Object[] os = {component};
+
+ logger.info(msg, os);
+ }
+
+ @Override
+ public void info(String msg, Object[] os, Throwable t) {
+ os = addProcessorAndThrowable(os, t);
+ msg = "{} " + msg + ": {}";
+
+ logger.info(msg, os);
+ if (logger.isDebugEnabled()) {
+ logger.info("", t);
+ }
+ }
+
+ @Override
+ public String getName() {
+ return logger.getName();
+ }
+
+ @Override
+ public void error(String msg, Throwable t) {
+ msg = "{} " + msg;
+ final Object[] os = {component};
+
+ logger.error(msg, os, t);
+ if (logger.isDebugEnabled()) {
+ logger.error("", t);
+ }
+ }
+
+ @Override
+ public void error(String msg, Object[] os) {
+ if (lastArgIsException(os)) {
+ error(msg, translateException(os), (Throwable) os[os.length - 1]);
+ } else {
+ os = addProcessor(os);
+ msg = "{} " + msg;
+ logger.error(msg, os);
+ }
+ }
+
+ @Override
+ public void error(String msg) {
+ msg = "{} " + msg;
+ final Object[] os = {component};
+
+ logger.error(msg, os);
+ }
+
+ @Override
+ public void error(String msg, Object[] os, Throwable t) {
+ os = addProcessorAndThrowable(os, t);
+ msg = "{} " + msg + ": {}";
+
+ logger.error(msg, os);
+ if (logger.isDebugEnabled()) {
+ logger.error("", t);
+ }
+ }
+
+ @Override
+ public void debug(String msg, Throwable t) {
+ msg = "{} " + msg;
+ final Object[] os = {component};
+
+ logger.debug(msg, os, t);
+ }
+
+ @Override
+ public void debug(String msg, Object[] os) {
+ os = addProcessor(os);
+ msg = "{} " + msg;
+
+ logger.debug(msg, os);
+ }
+
+ @Override
+ public void debug(String msg, Object[] os, Throwable t) {
+ os = addProcessorAndThrowable(os, t);
+ msg = "{} " + msg + ": {}";
+
+ logger.debug(msg, os);
+ if (logger.isDebugEnabled()) {
+ logger.debug("", t);
+ }
+ }
+
+ @Override
+ public void debug(String msg) {
+ msg = "{} " + msg;
+ final Object[] os = {component};
+
+ logger.debug(msg, os);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-mock/src/main/java/org/apache/nifi/util/MockPropertyValue.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/MockPropertyValue.java b/nifi-mock/src/main/java/org/apache/nifi/util/MockPropertyValue.java
new file mode 100644
index 0000000..12436d4
--- /dev/null
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/MockPropertyValue.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.nifi.attribute.expression.language.Query;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.PropertyValue;
+import org.apache.nifi.controller.ControllerService;
+import org.apache.nifi.controller.ControllerServiceLookup;
+import org.apache.nifi.expression.AttributeValueDecorator;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.DataUnit;
+import org.apache.nifi.processor.exception.ProcessException;
+
+public class MockPropertyValue implements PropertyValue {
+
+ private final String rawValue;
+ private final Boolean expectExpressions;
+ private final ControllerServiceLookup serviceLookup;
+ private final PropertyDescriptor propertyDescriptor;
+ private boolean expressionsEvaluated = false;
+
+ public MockPropertyValue(final String rawValue, final ControllerServiceLookup serviceLookup) {
+ this(rawValue, serviceLookup, null);
+ }
+
+ public MockPropertyValue(final String rawValue, final ControllerServiceLookup serviceLookup, final PropertyDescriptor propertyDescriptor) {
+ this.rawValue = rawValue;
+ this.serviceLookup = serviceLookup;
+ this.expectExpressions = propertyDescriptor == null ? null : propertyDescriptor.isExpressionLanguageSupported();
+ this.propertyDescriptor = propertyDescriptor;
+ }
+
+ private void ensureExpressionsEvaluated() {
+ if (Boolean.TRUE.equals(expectExpressions) && !expressionsEvaluated) {
+ throw new IllegalStateException("Attempting to retrieve value of " + propertyDescriptor
+ + " without first evaluating Expressions, even though the PropertyDescriptor indicates "
+ + "that the Expression Language is Supported. If you realize that this is the case and do not want "
+ + "this error to occur, it can be disabled by calling TestRunner.setValidateExpressionUsage(false)");
+ }
+ }
+
+ @Override
+ public String getValue() {
+ ensureExpressionsEvaluated();
+ return rawValue;
+ }
+
+ @Override
+ public Integer asInteger() {
+ ensureExpressionsEvaluated();
+ return (rawValue == null) ? null : Integer.parseInt(rawValue.trim());
+ }
+
+ @Override
+ public Long asLong() {
+ ensureExpressionsEvaluated();
+ return (rawValue == null) ? null : Long.parseLong(rawValue.trim());
+ }
+
+ @Override
+ public Boolean asBoolean() {
+ ensureExpressionsEvaluated();
+ return (rawValue == null) ? null : Boolean.parseBoolean(rawValue.trim());
+ }
+
+ @Override
+ public Float asFloat() {
+ ensureExpressionsEvaluated();
+ return (rawValue == null) ? null : Float.parseFloat(rawValue.trim());
+ }
+
+ @Override
+ public Double asDouble() {
+ ensureExpressionsEvaluated();
+ return (rawValue == null) ? null : Double.parseDouble(rawValue.trim());
+ }
+
+ @Override
+ public Long asTimePeriod(final TimeUnit timeUnit) {
+ ensureExpressionsEvaluated();
+ return (rawValue == null) ? null : FormatUtils.getTimeDuration(rawValue.trim(), timeUnit);
+ }
+
+ @Override
+ public Double asDataSize(final DataUnit dataUnit) {
+ ensureExpressionsEvaluated();
+ return rawValue == null ? null : DataUnit.parseDataSize(rawValue.trim(), dataUnit);
+ }
+
+ private void markEvaluated() {
+ if (Boolean.FALSE.equals(expectExpressions)) {
+ throw new IllegalStateException("Attempting to Evaluate Expressions but " + propertyDescriptor
+ + " indicates that the Expression Language is not supported. If you realize that this is the case and do not want "
+ + "this error to occur, it can be disabled by calling TestRunner.setValidateExpressionUsage(false)");
+ }
+ expressionsEvaluated = true;
+ }
+
+ @Override
+ public PropertyValue evaluateAttributeExpressions() throws ProcessException {
+ markEvaluated();
+ if (rawValue == null) {
+ return this;
+ }
+ return evaluateAttributeExpressions(null, null);
+ }
+
+ @Override
+ public PropertyValue evaluateAttributeExpressions(final AttributeValueDecorator decorator) throws ProcessException {
+ markEvaluated();
+ if (rawValue == null) {
+ return this;
+ }
+ return evaluateAttributeExpressions(null, decorator);
+ }
+
+ @Override
+ public PropertyValue evaluateAttributeExpressions(final FlowFile flowFile) throws ProcessException {
+ markEvaluated();
+ if (rawValue == null) {
+ return this;
+ }
+ return evaluateAttributeExpressions(flowFile, null);
+ }
+
+ @Override
+ public PropertyValue evaluateAttributeExpressions(final FlowFile flowFile, final AttributeValueDecorator decorator) throws ProcessException {
+ markEvaluated();
+ if (rawValue == null) {
+ return this;
+ }
+ return new MockPropertyValue(Query.evaluateExpressions(rawValue, flowFile, decorator), serviceLookup);
+ }
+
+ @Override
+ public ControllerService asControllerService() {
+ ensureExpressionsEvaluated();
+ if (rawValue == null || rawValue.equals("")) {
+ return null;
+ }
+
+ return serviceLookup.getControllerService(rawValue);
+ }
+
+ @Override
+ public <T extends ControllerService> T asControllerService(final Class<T> serviceType) throws IllegalArgumentException {
+ ensureExpressionsEvaluated();
+ if (rawValue == null || rawValue.equals("")) {
+ return null;
+ }
+
+ final ControllerService service = serviceLookup.getControllerService(rawValue);
+ if (serviceType.isAssignableFrom(service.getClass())) {
+ return serviceType.cast(service);
+ }
+ throw new IllegalArgumentException("Controller Service with identifier " + rawValue + " is of type " + service.getClass() + " and cannot be cast to " + serviceType);
+ }
+
+ @Override
+ public boolean isSet() {
+ return rawValue != null;
+ }
+
+ @Override
+ public String toString() {
+ return getValue();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-mock/src/main/java/org/apache/nifi/util/MockProvenanceReporter.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/MockProvenanceReporter.java b/nifi-mock/src/main/java/org/apache/nifi/util/MockProvenanceReporter.java
new file mode 100644
index 0000000..8c9a320
--- /dev/null
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/MockProvenanceReporter.java
@@ -0,0 +1,453 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.FlowFileHandlingException;
+import org.apache.nifi.provenance.ProvenanceEventBuilder;
+import org.apache.nifi.provenance.ProvenanceEventRecord;
+import org.apache.nifi.provenance.ProvenanceEventType;
+import org.apache.nifi.provenance.ProvenanceReporter;
+import org.apache.nifi.provenance.StandardProvenanceEventRecord;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class MockProvenanceReporter implements ProvenanceReporter {
+ private static final Logger logger = LoggerFactory.getLogger(MockProvenanceReporter.class);
+ private final MockProcessSession session;
+ private final String processorId;
+ private final String processorType;
+ private final SharedSessionState sharedSessionState;
+ private final Set<ProvenanceEventRecord> events = new LinkedHashSet<>();
+
+ public MockProvenanceReporter(final MockProcessSession session, final SharedSessionState sharedState, final String processorId, final String processorType) {
+ this.session = session;
+ this.sharedSessionState = sharedState;
+ this.processorId = processorId;
+ this.processorType = processorType;
+ }
+
+ private void verifyFlowFileKnown(final FlowFile flowFile) {
+ if (session != null && !session.isFlowFileKnown(flowFile)) {
+ throw new FlowFileHandlingException(flowFile + " is not known to " + session);
+ }
+ }
+
+ Set<ProvenanceEventRecord> getEvents() {
+ return Collections.unmodifiableSet(events);
+ }
+
+ /**
+ * Removes the given event from the reporter
+ *
+ * @param event
+ * event
+ */
+ void remove(final ProvenanceEventRecord event) {
+ events.remove(event);
+ }
+
+ void clear() {
+ events.clear();
+ }
+
+ /**
+ * Generates a Fork event for the given child and parents but does not
+ * register the event. This is useful so that a ProcessSession has the
+ * ability to de-dupe events, since one or more events may be created by the
+ * session itself, as well as by the Processor
+ *
+ * @param parents
+ * parents
+ * @param child
+ * child
+ * @return record
+ */
+ ProvenanceEventRecord generateJoinEvent(final Collection<FlowFile> parents, final FlowFile child) {
+ final ProvenanceEventBuilder eventBuilder = build(child, ProvenanceEventType.JOIN);
+ eventBuilder.addChildFlowFile(child);
+
+ for (final FlowFile parent : parents) {
+ eventBuilder.addParentFlowFile(parent);
+ }
+
+ return eventBuilder.build();
+ }
+
+ ProvenanceEventRecord generateDropEvent(final FlowFile flowFile, final String details) {
+ return build(flowFile, ProvenanceEventType.DROP).setDetails(details).build();
+ }
+
+ @Override
+ public void receive(final FlowFile flowFile, final String transitUri) {
+ receive(flowFile, transitUri, -1L);
+ }
+
+ @Override
+ public void receive(FlowFile flowFile, String transitUri, String sourceSystemFlowFileIdentifier) {
+ receive(flowFile, transitUri, sourceSystemFlowFileIdentifier, -1L);
+ }
+
+ @Override
+ public void receive(final FlowFile flowFile, final String transitUri, final long transmissionMillis) {
+ receive(flowFile, transitUri, null, transmissionMillis);
+ }
+
+ @Override
+ public void receive(final FlowFile flowFile, final String transitUri, final String sourceSystemFlowFileIdentifier, final long transmissionMillis) {
+ receive(flowFile, transitUri, sourceSystemFlowFileIdentifier, null, transmissionMillis);
+ }
+
+ @Override
+ public void receive(final FlowFile flowFile, final String transitUri, final String sourceSystemFlowFileIdentifier, final String details, final long transmissionMillis) {
+ verifyFlowFileKnown(flowFile);
+
+ try {
+ final ProvenanceEventRecord record = build(flowFile, ProvenanceEventType.RECEIVE)
+ .setTransitUri(transitUri).setSourceSystemFlowFileIdentifier(sourceSystemFlowFileIdentifier).setEventDuration(transmissionMillis).setDetails(details).build();
+ events.add(record);
+ } catch (final Exception e) {
+ logger.error("Failed to generate Provenance Event due to " + e);
+ if (logger.isDebugEnabled()) {
+ logger.error("", e);
+ }
+ }
+ }
+
+ @Override
+ public void send(final FlowFile flowFile, final String transitUri, final long transmissionMillis) {
+ send(flowFile, transitUri, transmissionMillis, true);
+ }
+
+ @Override
+ public void send(final FlowFile flowFile, final String transitUri) {
+ send(flowFile, transitUri, null, -1L, true);
+ }
+
+ @Override
+ public void send(final FlowFile flowFile, final String transitUri, final String details) {
+ send(flowFile, transitUri, details, -1L, true);
+ }
+
+ @Override
+ public void send(final FlowFile flowFile, final String transitUri, final long transmissionMillis, final boolean force) {
+ send(flowFile, transitUri, null, transmissionMillis, force);
+ }
+
+ @Override
+ public void send(final FlowFile flowFile, final String transitUri, final String details, final boolean force) {
+ send(flowFile, transitUri, details, -1L, force);
+ }
+
+ @Override
+ public void send(final FlowFile flowFile, final String transitUri, final String details, final long transmissionMillis) {
+ send(flowFile, transitUri, details, transmissionMillis, true);
+ }
+
+ @Override
+ public void send(final FlowFile flowFile, final String transitUri, final String details, final long transmissionMillis, final boolean force) {
+ try {
+ final ProvenanceEventRecord record = build(flowFile, ProvenanceEventType.SEND).setTransitUri(transitUri).setEventDuration(transmissionMillis).setDetails(details).build();
+ if (force) {
+ sharedSessionState.addProvenanceEvents(Collections.singleton(record));
+ } else {
+ events.add(record);
+ }
+ } catch (final Exception e) {
+ logger.error("Failed to generate Provenance Event due to " + e);
+ if (logger.isDebugEnabled()) {
+ logger.error("", e);
+ }
+ }
+ }
+
+ @Override
+ public void send(final FlowFile flowFile, final String transitUri, final boolean force) {
+ send(flowFile, transitUri, -1L, true);
+ }
+
+ @Override
+ public void associate(final FlowFile flowFile, final String alternateIdentifierNamespace, final String alternateIdentifier) {
+ try {
+ String trimmedNamespace = alternateIdentifierNamespace.trim();
+ if (trimmedNamespace.endsWith(":")) {
+ trimmedNamespace = trimmedNamespace.substring(0, trimmedNamespace.length() - 1);
+ }
+
+ String trimmedIdentifier = alternateIdentifier.trim();
+ if (trimmedIdentifier.startsWith(":")) {
+ if (trimmedIdentifier.length() == 1) {
+ throw new IllegalArgumentException("Illegal alternateIdentifier: " + alternateIdentifier);
+ }
+ trimmedIdentifier = trimmedIdentifier.substring(1);
+ }
+
+ final String alternateIdentifierUri = trimmedNamespace + ":" + trimmedIdentifier;
+ final ProvenanceEventRecord record = build(flowFile, ProvenanceEventType.ADDINFO).setAlternateIdentifierUri(alternateIdentifierUri).build();
+ events.add(record);
+ } catch (final Exception e) {
+ logger.error("Failed to generate Provenance Event due to " + e);
+ if (logger.isDebugEnabled()) {
+ logger.error("", e);
+ }
+ }
+ }
+
+ ProvenanceEventRecord drop(final FlowFile flowFile, final String reason) {
+ try {
+ final ProvenanceEventBuilder builder = build(flowFile, ProvenanceEventType.DROP);
+ if (reason != null) {
+ builder.setDetails("Discard reason: " + reason);
+ }
+ final ProvenanceEventRecord record = builder.build();
+ events.add(record);
+ return record;
+ } catch (final Exception e) {
+ logger.error("Failed to generate Provenance Event due to " + e);
+ if (logger.isDebugEnabled()) {
+ logger.error("", e);
+ }
+ return null;
+ }
+ }
+
+ void expire(final FlowFile flowFile, final String details) {
+ try {
+ final ProvenanceEventRecord record = build(flowFile, ProvenanceEventType.EXPIRE).setDetails(details).build();
+ events.add(record);
+ } catch (final Exception e) {
+ logger.error("Failed to generate Provenance Event due to " + e);
+ if (logger.isDebugEnabled()) {
+ logger.error("", e);
+ }
+ }
+ }
+
+ @Override
+ public void fork(final FlowFile parent, final Collection<FlowFile> children) {
+ fork(parent, children, null, -1L);
+ }
+
+ @Override
+ public void fork(final FlowFile parent, final Collection<FlowFile> children, final long forkDuration) {
+ fork(parent, children, null, forkDuration);
+ }
+
+ @Override
+ public void fork(final FlowFile parent, final Collection<FlowFile> children, final String details) {
+ fork(parent, children, details, -1L);
+ }
+
+ @Override
+ public void fork(final FlowFile parent, final Collection<FlowFile> children, final String details, final long forkDuration) {
+ verifyFlowFileKnown(parent);
+
+ try {
+ final ProvenanceEventBuilder eventBuilder = build(parent, ProvenanceEventType.FORK);
+ eventBuilder.addParentFlowFile(parent);
+ for (final FlowFile child : children) {
+ eventBuilder.addChildFlowFile(child);
+ }
+
+ if (forkDuration > -1L) {
+ eventBuilder.setEventDuration(forkDuration);
+ }
+
+ if (details != null) {
+ eventBuilder.setDetails(details);
+ }
+
+ events.add(eventBuilder.build());
+ } catch (final Exception e) {
+ logger.error("Failed to generate Provenance Event due to " + e);
+ if (logger.isDebugEnabled()) {
+ logger.error("", e);
+ }
+ }
+ }
+
+ @Override
+ public void join(final Collection<FlowFile> parents, final FlowFile child) {
+ join(parents, child, null, -1L);
+ }
+
+ @Override
+ public void join(final Collection<FlowFile> parents, final FlowFile child, final long joinDuration) {
+ join(parents, child, null, joinDuration);
+ }
+
+ @Override
+ public void join(final Collection<FlowFile> parents, final FlowFile child, final String details) {
+ join(parents, child, details, -1L);
+ }
+
+ @Override
+ public void join(final Collection<FlowFile> parents, final FlowFile child, final String details, final long joinDuration) {
+ verifyFlowFileKnown(child);
+
+ try {
+ final ProvenanceEventBuilder eventBuilder = build(child, ProvenanceEventType.JOIN);
+ eventBuilder.addChildFlowFile(child);
+ eventBuilder.setDetails(details);
+
+ for (final FlowFile parent : parents) {
+ eventBuilder.addParentFlowFile(parent);
+ }
+
+ events.add(eventBuilder.build());
+ } catch (final Exception e) {
+ logger.error("Failed to generate Provenance Event due to " + e);
+ if (logger.isDebugEnabled()) {
+ logger.error("", e);
+ }
+ }
+ }
+
+ @Override
+ public void clone(final FlowFile parent, final FlowFile child) {
+ verifyFlowFileKnown(child);
+
+ try {
+ final ProvenanceEventBuilder eventBuilder = build(parent, ProvenanceEventType.CLONE);
+ eventBuilder.addChildFlowFile(child);
+ eventBuilder.addParentFlowFile(parent);
+ events.add(eventBuilder.build());
+ } catch (final Exception e) {
+ logger.error("Failed to generate Provenance Event due to " + e);
+ if (logger.isDebugEnabled()) {
+ logger.error("", e);
+ }
+ }
+ }
+
+ @Override
+ public void modifyContent(final FlowFile flowFile) {
+ modifyContent(flowFile, null, -1L);
+ }
+
+ @Override
+ public void modifyContent(final FlowFile flowFile, final String details) {
+ modifyContent(flowFile, details, -1L);
+ }
+
+ @Override
+ public void modifyContent(final FlowFile flowFile, final long processingMillis) {
+ modifyContent(flowFile, null, processingMillis);
+ }
+
+ @Override
+ public void modifyContent(final FlowFile flowFile, final String details, final long processingMillis) {
+ verifyFlowFileKnown(flowFile);
+
+ try {
+ final ProvenanceEventRecord record = build(flowFile, ProvenanceEventType.CONTENT_MODIFIED).setEventDuration(processingMillis).setDetails(details).build();
+ events.add(record);
+ } catch (final Exception e) {
+ logger.error("Failed to generate Provenance Event due to " + e);
+ if (logger.isDebugEnabled()) {
+ logger.error("", e);
+ }
+ }
+ }
+
+ @Override
+ public void modifyAttributes(final FlowFile flowFile) {
+ modifyAttributes(flowFile, null);
+ }
+
+ @Override
+ public void modifyAttributes(final FlowFile flowFile, final String details) {
+ verifyFlowFileKnown(flowFile);
+
+ try {
+ final ProvenanceEventRecord record = build(flowFile, ProvenanceEventType.ATTRIBUTES_MODIFIED).setDetails(details).build();
+ events.add(record);
+ } catch (final Exception e) {
+ logger.error("Failed to generate Provenance Event due to " + e);
+ if (logger.isDebugEnabled()) {
+ logger.error("", e);
+ }
+ }
+ }
+
+ @Override
+ public void route(final FlowFile flowFile, final Relationship relationship) {
+ route(flowFile, relationship, null);
+ }
+
+ @Override
+ public void route(final FlowFile flowFile, final Relationship relationship, final long processingDuration) {
+ route(flowFile, relationship, null, processingDuration);
+ }
+
+ @Override
+ public void route(final FlowFile flowFile, final Relationship relationship, final String details) {
+ route(flowFile, relationship, details, -1L);
+ }
+
+ @Override
+ public void route(final FlowFile flowFile, final Relationship relationship, final String details, final long processingDuration) {
+ verifyFlowFileKnown(flowFile);
+
+ try {
+ final ProvenanceEventRecord record = build(flowFile, ProvenanceEventType.ROUTE).setRelationship(relationship).setDetails(details).setEventDuration(processingDuration).build();
+ events.add(record);
+ } catch (final Exception e) {
+ logger.error("Failed to generate Provenance Event due to " + e);
+ if (logger.isDebugEnabled()) {
+ logger.error("", e);
+ }
+ }
+ }
+
+ @Override
+ public void create(final FlowFile flowFile) {
+ create(flowFile, null);
+ }
+
+ @Override
+ public void create(final FlowFile flowFile, final String details) {
+ verifyFlowFileKnown(flowFile);
+
+ try {
+ final ProvenanceEventRecord record = build(flowFile, ProvenanceEventType.CREATE).setDetails(details).build();
+ events.add(record);
+ } catch (final Exception e) {
+ logger.error("Failed to generate Provenance Event due to " + e);
+ if (logger.isDebugEnabled()) {
+ logger.error("", e);
+ }
+ }
+ }
+
+ ProvenanceEventBuilder build(final FlowFile flowFile, final ProvenanceEventType eventType) {
+ final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
+ builder.setEventType(eventType);
+ builder.fromFlowFile(flowFile);
+ builder.setLineageStartDate(flowFile.getLineageStartDate());
+ builder.setComponentId(processorId);
+ builder.setComponentType(processorType);
+ return builder;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-mock/src/main/java/org/apache/nifi/util/MockReportingContext.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/MockReportingContext.java b/nifi-mock/src/main/java/org/apache/nifi/util/MockReportingContext.java
new file mode 100644
index 0000000..63a9876
--- /dev/null
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/MockReportingContext.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.PropertyValue;
+import org.apache.nifi.controller.ControllerService;
+import org.apache.nifi.controller.ControllerServiceLookup;
+import org.apache.nifi.reporting.Bulletin;
+import org.apache.nifi.reporting.BulletinFactory;
+import org.apache.nifi.reporting.BulletinRepository;
+import org.apache.nifi.reporting.ReportingContext;
+import org.apache.nifi.reporting.Severity;
+
+public class MockReportingContext extends MockControllerServiceLookup implements ReportingContext, ControllerServiceLookup {
+
+ private final Map<String, ControllerServiceConfiguration> controllerServices;
+ private final MockEventAccess eventAccess = new MockEventAccess();
+ private final Map<PropertyDescriptor, String> properties = new HashMap<>();
+
+ private final Map<String, List<Bulletin>> componentBulletinsCreated = new HashMap<>();
+
+ public MockReportingContext(final Map<String, ControllerService> controllerServices) {
+ this.controllerServices = new HashMap<>();
+ for (final Map.Entry<String, ControllerService> entry : controllerServices.entrySet()) {
+ this.controllerServices.put(entry.getKey(), new ControllerServiceConfiguration(entry.getValue()));
+ }
+ }
+
+ @Override
+ public Map<PropertyDescriptor, String> getProperties() {
+ return Collections.unmodifiableMap(properties);
+ }
+
+ @Override
+ public PropertyValue getProperty(final PropertyDescriptor property) {
+ final String configuredValue = properties.get(property);
+ return new MockPropertyValue(configuredValue == null ? property.getDefaultValue() : configuredValue, this);
+ }
+
+ public void setProperty(final String propertyName, final String value) {
+ this.properties.put(new PropertyDescriptor.Builder().name(propertyName).build(), value);
+ }
+
+ public void setProperties(final Map<PropertyDescriptor, String> properties) {
+ this.properties.clear();
+ this.properties.putAll(properties);
+ }
+
+ @Override
+ public MockEventAccess getEventAccess() {
+ return eventAccess;
+ }
+
+ @Override
+ public BulletinRepository getBulletinRepository() {
+ return new MockBulletinRepository();
+ }
+
+ @Override
+ public Bulletin createBulletin(final String category, final Severity severity, final String message) {
+ return BulletinFactory.createBulletin(category, severity.name(), message);
+ }
+
+ @Override
+ public Bulletin createBulletin(final String componentId, final String category, final Severity severity, final String message) {
+ final Bulletin bulletin = BulletinFactory.createBulletin(null, componentId, "test processor", category, severity.name(), message);
+ List<Bulletin> bulletins = componentBulletinsCreated.get(componentId);
+ if (bulletins == null) {
+ bulletins = new ArrayList<>();
+ componentBulletinsCreated.put(componentId, bulletins);
+ }
+ bulletins.add(bulletin);
+ return bulletin;
+ }
+
+ @Override
+ public ControllerServiceLookup getControllerServiceLookup() {
+ return this;
+ }
+
+ /**
+ * @param componentId identifier of component to get bulletins for
+ * @return all Bulletins that have been created for the component with the
+ * given ID
+ */
+ public List<Bulletin> getComponentBulletins(final String componentId) {
+ final List<Bulletin> created = componentBulletinsCreated.get(componentId);
+ if (created == null) {
+ return new ArrayList<>();
+ }
+
+ return new ArrayList<>(created);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-mock/src/main/java/org/apache/nifi/util/MockReportingInitializationContext.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/MockReportingInitializationContext.java b/nifi-mock/src/main/java/org/apache/nifi/util/MockReportingInitializationContext.java
new file mode 100644
index 0000000..0aea00a
--- /dev/null
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/MockReportingInitializationContext.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.controller.ControllerServiceLookup;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.reporting.ReportingInitializationContext;
+import org.apache.nifi.scheduling.SchedulingStrategy;
+
+public class MockReportingInitializationContext extends MockControllerServiceLookup implements ReportingInitializationContext, ControllerServiceLookup {
+
+ private final String identifier;
+ private final String name;
+ private final Map<PropertyDescriptor, String> properties = new HashMap<>();
+ private final ComponentLog logger;
+
+ public MockReportingInitializationContext(final String identifier, final String name, final ComponentLog logger) {
+ this.identifier = identifier;
+ this.name = name;
+ this.logger = logger;
+ }
+
+ @Override
+ public String getIdentifier() {
+ return identifier;
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public long getSchedulingPeriod(final TimeUnit timeUnit) {
+ return 1L;
+ }
+
+ public void setProperty(final String propertyName, final String value) {
+ setProperty(new PropertyDescriptor.Builder().name(propertyName).build(), value);
+ }
+
+ public void setProperty(final PropertyDescriptor propertyName, final String value) {
+ this.properties.put(propertyName, value);
+ }
+
+ public void setProperties(final Map<PropertyDescriptor, String> properties) {
+ this.properties.clear();
+ this.properties.putAll(properties);
+ }
+
+ @Override
+ public ControllerServiceLookup getControllerServiceLookup() {
+ return this;
+ }
+
+ @Override
+ public String getSchedulingPeriod() {
+ return "0 sec";
+ }
+
+ @Override
+ public SchedulingStrategy getSchedulingStrategy() {
+ return SchedulingStrategy.TIMER_DRIVEN;
+ }
+
+ @Override
+ public ComponentLog getLogger() {
+ return logger;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-mock/src/main/java/org/apache/nifi/util/MockSessionFactory.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/MockSessionFactory.java b/nifi-mock/src/main/java/org/apache/nifi/util/MockSessionFactory.java
new file mode 100644
index 0000000..49b8796
--- /dev/null
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/MockSessionFactory.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import java.util.Collections;
+import java.util.Set;
+import java.util.concurrent.CopyOnWriteArraySet;
+
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.ProcessSessionFactory;
+import org.apache.nifi.processor.Processor;
+
+public class MockSessionFactory implements ProcessSessionFactory {
+
+ private final Processor processor;
+ private final SharedSessionState sharedState;
+ private final Set<MockProcessSession> createdSessions = new CopyOnWriteArraySet<>();
+
+ MockSessionFactory(final SharedSessionState sharedState, final Processor processor) {
+ this.sharedState = sharedState;
+ this.processor = processor;
+ }
+
+ @Override
+ public ProcessSession createSession() {
+ final MockProcessSession session = new MockProcessSession(sharedState, processor);
+ createdSessions.add(session);
+ return session;
+ }
+
+ Set<MockProcessSession> getCreatedSessions() {
+ return Collections.unmodifiableSet(createdSessions);
+ }
+}
[30/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java
new file mode 100644
index 0000000..e25f5d6
--- /dev/null
+++ b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java
@@ -0,0 +1,932 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.net.InetSocketAddress;
+import java.nio.file.InvalidPathException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class NiFiProperties extends Properties {
+
+ private static final long serialVersionUID = 2119177359005492702L;
+
+ private static final Logger LOG = LoggerFactory.getLogger(NiFiProperties.class);
+ private static NiFiProperties instance = null;
+
+ // core properties
+ public static final String PROPERTIES_FILE_PATH = "nifi.properties.file.path";
+ public static final String FLOW_CONFIGURATION_FILE = "nifi.flow.configuration.file";
+ public static final String FLOW_CONFIGURATION_ARCHIVE_FILE = "nifi.flow.configuration.archive.file";
+ public static final String AUTHORITY_PROVIDER_CONFIGURATION_FILE = "nifi.authority.provider.configuration.file";
+ public static final String REPOSITORY_DATABASE_DIRECTORY = "nifi.database.directory";
+ public static final String RESTORE_DIRECTORY = "nifi.restore.directory";
+ public static final String VERSION = "nifi.version";
+ public static final String WRITE_DELAY_INTERVAL = "nifi.flowservice.writedelay.interval";
+ public static final String AUTO_RESUME_STATE = "nifi.flowcontroller.autoResumeState";
+ public static final String FLOW_CONTROLLER_GRACEFUL_SHUTDOWN_PERIOD = "nifi.flowcontroller.graceful.shutdown.period";
+ public static final String NAR_LIBRARY_DIRECTORY = "nifi.nar.library.directory";
+ public static final String NAR_LIBRARY_DIRECTORY_PREFIX = "nifi.nar.library.directory.";
+ public static final String NAR_WORKING_DIRECTORY = "nifi.nar.working.directory";
+ public static final String COMPONENT_DOCS_DIRECTORY = "nifi.documentation.working.directory";
+ public static final String SENSITIVE_PROPS_KEY = "nifi.sensitive.props.key";
+ public static final String SENSITIVE_PROPS_ALGORITHM = "nifi.sensitive.props.algorithm";
+ public static final String SENSITIVE_PROPS_PROVIDER = "nifi.sensitive.props.provider";
+ public static final String H2_URL_APPEND = "nifi.h2.url.append";
+ public static final String REMOTE_INPUT_PORT = "nifi.remote.input.socket.port";
+ public static final String SITE_TO_SITE_SECURE = "nifi.remote.input.secure";
+ public static final String TEMPLATE_DIRECTORY = "nifi.templates.directory";
+ public static final String ADMINISTRATIVE_YIELD_DURATION = "nifi.administrative.yield.duration";
+ public static final String PERSISTENT_STATE_DIRECTORY = "nifi.persistent.state.directory";
+ public static final String BORED_YIELD_DURATION = "nifi.bored.yield.duration";
+
+ // content repository properties
+ public static final String REPOSITORY_CONTENT_PREFIX = "nifi.content.repository.directory.";
+ public static final String CONTENT_REPOSITORY_IMPLEMENTATION = "nifi.content.repository.implementation";
+ public static final String MAX_APPENDABLE_CLAIM_SIZE = "nifi.content.claim.max.appendable.size";
+ public static final String MAX_FLOWFILES_PER_CLAIM = "nifi.content.claim.max.flow.files";
+ public static final String CONTENT_ARCHIVE_MAX_RETENTION_PERIOD = "nifi.content.repository.archive.max.retention.period";
+ public static final String CONTENT_ARCHIVE_MAX_USAGE_PERCENTAGE = "nifi.content.repository.archive.max.usage.percentage";
+ public static final String CONTENT_ARCHIVE_BACK_PRESSURE_PERCENTAGE = "nifi.content.repository.archive.backpressure.percentage";
+ public static final String CONTENT_ARCHIVE_ENABLED = "nifi.content.repository.archive.enabled";
+ public static final String CONTENT_ARCHIVE_CLEANUP_FREQUENCY = "nifi.content.repository.archive.cleanup.frequency";
+ public static final String CONTENT_VIEWER_URL = "nifi.content.viewer.url";
+
+ // flowfile repository properties
+ public static final String FLOWFILE_REPOSITORY_IMPLEMENTATION = "nifi.flowfile.repository.implementation";
+ public static final String FLOWFILE_REPOSITORY_ALWAYS_SYNC = "nifi.flowfile.repository.always.sync";
+ public static final String FLOWFILE_REPOSITORY_DIRECTORY = "nifi.flowfile.repository.directory";
+ public static final String FLOWFILE_REPOSITORY_PARTITIONS = "nifi.flowfile.repository.partitions";
+ public static final String FLOWFILE_REPOSITORY_CHECKPOINT_INTERVAL = "nifi.flowfile.repository.checkpoint.interval";
+ public static final String FLOWFILE_SWAP_MANAGER_IMPLEMENTATION = "nifi.swap.manager.implementation";
+ public static final String QUEUE_SWAP_THRESHOLD = "nifi.queue.swap.threshold";
+ public static final String SWAP_IN_THREADS = "nifi.swap.in.threads";
+ public static final String SWAP_IN_PERIOD = "nifi.swap.in.period";
+ public static final String SWAP_OUT_THREADS = "nifi.swap.out.threads";
+ public static final String SWAP_OUT_PERIOD = "nifi.swap.out.period";
+
+ // provenance properties
+ public static final String PROVENANCE_REPO_IMPLEMENTATION_CLASS = "nifi.provenance.repository.implementation";
+ public static final String PROVENANCE_REPO_DIRECTORY_PREFIX = "nifi.provenance.repository.directory.";
+ public static final String PROVENANCE_MAX_STORAGE_TIME = "nifi.provenance.repository.max.storage.time";
+ public static final String PROVENANCE_MAX_STORAGE_SIZE = "nifi.provenance.repository.max.storage.size";
+ public static final String PROVENANCE_ROLLOVER_TIME = "nifi.provenance.repository.rollover.time";
+ public static final String PROVENANCE_ROLLOVER_SIZE = "nifi.provenance.repository.rollover.size";
+ public static final String PROVENANCE_QUERY_THREAD_POOL_SIZE = "nifi.provenance.repository.query.threads";
+ public static final String PROVENANCE_COMPRESS_ON_ROLLOVER = "nifi.provenance.repository.compress.on.rollover";
+ public static final String PROVENANCE_INDEXED_FIELDS = "nifi.provenance.repository.indexed.fields";
+ public static final String PROVENANCE_INDEXED_ATTRIBUTES = "nifi.provenance.repository.indexed.attributes";
+ public static final String PROVENANCE_INDEX_SHARD_SIZE = "nifi.provenance.repository.index.shard.size";
+ public static final String PROVENANCE_JOURNAL_COUNT = "nifi.provenance.repository.journal.count";
+
+ // component status repository properties
+ public static final String COMPONENT_STATUS_REPOSITORY_IMPLEMENTATION = "nifi.components.status.repository.implementation";
+ public static final String COMPONENT_STATUS_SNAPSHOT_FREQUENCY = "nifi.components.status.snapshot.frequency";
+
+ // encryptor properties
+ public static final String NF_SENSITIVE_PROPS_KEY = "nifi.sensitive.props.key";
+ public static final String NF_SENSITIVE_PROPS_ALGORITHM = "nifi.sensitive.props.algorithm";
+ public static final String NF_SENSITIVE_PROPS_PROVIDER = "nifi.sensitive.props.provider";
+
+ // security properties
+ public static final String SECURITY_KEYSTORE = "nifi.security.keystore";
+ public static final String SECURITY_KEYSTORE_TYPE = "nifi.security.keystoreType";
+ public static final String SECURITY_KEYSTORE_PASSWD = "nifi.security.keystorePasswd";
+ public static final String SECURITY_KEY_PASSWD = "nifi.security.keyPasswd";
+ public static final String SECURITY_TRUSTSTORE = "nifi.security.truststore";
+ public static final String SECURITY_TRUSTSTORE_TYPE = "nifi.security.truststoreType";
+ public static final String SECURITY_TRUSTSTORE_PASSWD = "nifi.security.truststorePasswd";
+ public static final String SECURITY_NEED_CLIENT_AUTH = "nifi.security.needClientAuth";
+ public static final String SECURITY_USER_AUTHORITY_PROVIDER = "nifi.security.user.authority.provider";
+ public static final String SECURITY_CLUSTER_AUTHORITY_PROVIDER_PORT = "nifi.security.cluster.authority.provider.port";
+ public static final String SECURITY_CLUSTER_AUTHORITY_PROVIDER_THREADS = "nifi.security.cluster.authority.provider.threads";
+ public static final String SECURITY_USER_CREDENTIAL_CACHE_DURATION = "nifi.security.user.credential.cache.duration";
+ public static final String SECURITY_SUPPORT_NEW_ACCOUNT_REQUESTS = "nifi.security.support.new.account.requests";
+ public static final String SECURITY_DEFAULT_USER_ROLES = "nifi.security.default.user.roles";
+ public static final String SECURITY_OCSP_RESPONDER_URL = "nifi.security.ocsp.responder.url";
+ public static final String SECURITY_OCSP_RESPONDER_CERTIFICATE = "nifi.security.ocsp.responder.certificate";
+
+ // web properties
+ public static final String WEB_WAR_DIR = "nifi.web.war.directory";
+ public static final String WEB_HTTP_PORT = "nifi.web.http.port";
+ public static final String WEB_HTTP_HOST = "nifi.web.http.host";
+ public static final String WEB_HTTPS_PORT = "nifi.web.https.port";
+ public static final String WEB_HTTPS_HOST = "nifi.web.https.host";
+ public static final String WEB_WORKING_DIR = "nifi.web.jetty.working.directory";
+ public static final String WEB_THREADS = "nifi.web.jetty.threads";
+
+ // ui properties
+ public static final String UI_BANNER_TEXT = "nifi.ui.banner.text";
+ public static final String UI_AUTO_REFRESH_INTERVAL = "nifi.ui.autorefresh.interval";
+
+ // cluster common properties
+ public static final String CLUSTER_PROTOCOL_HEARTBEAT_INTERVAL = "nifi.cluster.protocol.heartbeat.interval";
+ public static final String CLUSTER_PROTOCOL_IS_SECURE = "nifi.cluster.protocol.is.secure";
+ public static final String CLUSTER_PROTOCOL_SOCKET_TIMEOUT = "nifi.cluster.protocol.socket.timeout";
+ public static final String CLUSTER_PROTOCOL_CONNECTION_HANDSHAKE_TIMEOUT = "nifi.cluster.protocol.connection.handshake.timeout";
+ public static final String CLUSTER_PROTOCOL_USE_MULTICAST = "nifi.cluster.protocol.use.multicast";
+ public static final String CLUSTER_PROTOCOL_MULTICAST_ADDRESS = "nifi.cluster.protocol.multicast.address";
+ public static final String CLUSTER_PROTOCOL_MULTICAST_PORT = "nifi.cluster.protocol.multicast.port";
+ public static final String CLUSTER_PROTOCOL_MULTICAST_SERVICE_BROADCAST_DELAY = "nifi.cluster.protocol.multicast.service.broadcast.delay";
+ public static final String CLUSTER_PROTOCOL_MULTICAST_SERVICE_LOCATOR_ATTEMPTS = "nifi.cluster.protocol.multicast.service.locator.attempts";
+ public static final String CLUSTER_PROTOCOL_MULTICAST_SERVICE_LOCATOR_ATTEMPTS_DELAY = "nifi.cluster.protocol.multicast.service.locator.attempts.delay";
+
+ // cluster node properties
+ public static final String CLUSTER_IS_NODE = "nifi.cluster.is.node";
+ public static final String CLUSTER_NODE_ADDRESS = "nifi.cluster.node.address";
+ public static final String CLUSTER_NODE_PROTOCOL_PORT = "nifi.cluster.node.protocol.port";
+ public static final String CLUSTER_NODE_PROTOCOL_THREADS = "nifi.cluster.node.protocol.threads";
+ public static final String CLUSTER_NODE_UNICAST_MANAGER_ADDRESS = "nifi.cluster.node.unicast.manager.address";
+ public static final String CLUSTER_NODE_UNICAST_MANAGER_PROTOCOL_PORT = "nifi.cluster.node.unicast.manager.protocol.port";
+
+ // cluster manager properties
+ public static final String CLUSTER_IS_MANAGER = "nifi.cluster.is.manager";
+ public static final String CLUSTER_MANAGER_ADDRESS = "nifi.cluster.manager.address";
+ public static final String CLUSTER_MANAGER_PROTOCOL_PORT = "nifi.cluster.manager.protocol.port";
+ public static final String CLUSTER_MANAGER_NODE_FIREWALL_FILE = "nifi.cluster.manager.node.firewall.file";
+ public static final String CLUSTER_MANAGER_NODE_EVENT_HISTORY_SIZE = "nifi.cluster.manager.node.event.history.size";
+ public static final String CLUSTER_MANAGER_NODE_API_CONNECTION_TIMEOUT = "nifi.cluster.manager.node.api.connection.timeout";
+ public static final String CLUSTER_MANAGER_NODE_API_READ_TIMEOUT = "nifi.cluster.manager.node.api.read.timeout";
+ public static final String CLUSTER_MANAGER_NODE_API_REQUEST_THREADS = "nifi.cluster.manager.node.api.request.threads";
+ public static final String CLUSTER_MANAGER_FLOW_RETRIEVAL_DELAY = "nifi.cluster.manager.flow.retrieval.delay";
+ public static final String CLUSTER_MANAGER_PROTOCOL_THREADS = "nifi.cluster.manager.protocol.threads";
+ public static final String CLUSTER_MANAGER_SAFEMODE_DURATION = "nifi.cluster.manager.safemode.duration";
+
+ // defaults
+ public static final String DEFAULT_TITLE = "NiFi";
+ public static final Boolean DEFAULT_AUTO_RESUME_STATE = true;
+ public static final String DEFAULT_AUTHORITY_PROVIDER_CONFIGURATION_FILE = "conf/authority-providers.xml";
+ public static final String DEFAULT_USER_CREDENTIAL_CACHE_DURATION = "24 hours";
+ public static final Integer DEFAULT_REMOTE_INPUT_PORT = null;
+ public static final Path DEFAULT_TEMPLATE_DIRECTORY = Paths.get("conf", "templates");
+ public static final int DEFAULT_WEB_THREADS = 200;
+ public static final String DEFAULT_WEB_WORKING_DIR = "./work/jetty";
+ public static final String DEFAULT_NAR_WORKING_DIR = "./work/nar";
+ public static final String DEFAULT_COMPONENT_DOCS_DIRECTORY = "./work/docs/components";
+ public static final String DEFAULT_NAR_LIBRARY_DIR = "./lib";
+ public static final String DEFAULT_FLOWFILE_REPO_PARTITIONS = "256";
+ public static final String DEFAULT_FLOWFILE_CHECKPOINT_INTERVAL = "2 min";
+ public static final int DEFAULT_MAX_FLOWFILES_PER_CLAIM = 100;
+ public static final int DEFAULT_QUEUE_SWAP_THRESHOLD = 20000;
+ public static final String DEFAULT_SWAP_STORAGE_LOCATION = "./flowfile_repository/swap";
+ public static final String DEFAULT_SWAP_IN_PERIOD = "1 sec";
+ public static final String DEFAULT_SWAP_OUT_PERIOD = "5 sec";
+ public static final int DEFAULT_SWAP_IN_THREADS = 4;
+ public static final int DEFAULT_SWAP_OUT_THREADS = 4;
+ public static final String DEFAULT_ADMINISTRATIVE_YIELD_DURATION = "30 sec";
+ public static final String DEFAULT_PERSISTENT_STATE_DIRECTORY = "./conf/state";
+ public static final String DEFAULT_COMPONENT_STATUS_SNAPSHOT_FREQUENCY = "5 mins";
+ public static final String DEFAULT_BORED_YIELD_DURATION = "10 millis";
+
+ // cluster common defaults
+ public static final String DEFAULT_CLUSTER_PROTOCOL_HEARTBEAT_INTERVAL = "5 sec";
+ public static final String DEFAULT_CLUSTER_PROTOCOL_MULTICAST_SERVICE_BROADCAST_DELAY = "500 ms";
+ public static final int DEFAULT_CLUSTER_PROTOCOL_MULTICAST_SERVICE_LOCATOR_ATTEMPTS = 3;
+ public static final String DEFAULT_CLUSTER_PROTOCOL_MULTICAST_SERVICE_LOCATOR_ATTEMPTS_DELAY = "1 sec";
+ public static final String DEFAULT_CLUSTER_PROTOCOL_SOCKET_TIMEOUT = "30 sec";
+ public static final String DEFAULT_CLUSTER_PROTOCOL_CONNECTION_HANDSHAKE_TIMEOUT = "45 sec";
+
+ // cluster node defaults
+ public static final int DEFAULT_CLUSTER_NODE_PROTOCOL_THREADS = 2;
+
+ // cluster manager defaults
+ public static final int DEFAULT_CLUSTER_MANAGER_NODE_EVENT_HISTORY_SIZE = 10;
+ public static final String DEFAULT_CLUSTER_MANAGER_NODE_API_CONNECTION_TIMEOUT = "30 sec";
+ public static final String DEFAULT_CLUSTER_MANAGER_NODE_API_READ_TIMEOUT = "30 sec";
+ public static final int DEFAULT_CLUSTER_MANAGER_NODE_API_NUM_REQUEST_THREADS = 10;
+ public static final String DEFAULT_CLUSTER_MANAGER_FLOW_RETRIEVAL_DELAY = "5 sec";
+ public static final int DEFAULT_CLUSTER_MANAGER_PROTOCOL_THREADS = 10;
+ public static final String DEFAULT_CLUSTER_MANAGER_SAFEMODE_DURATION = "0 sec";
+
+ private NiFiProperties() {
+ super();
+ }
+
+ /**
+ * This is the method through which the NiFiProperties object should be
+ * obtained.
+ *
+ * @return the NiFiProperties object to use
+ * @throws RuntimeException if unable to load properties file
+ */
+ public static synchronized NiFiProperties getInstance() {
+ if (null == instance) {
+ final NiFiProperties suspectInstance = new NiFiProperties();
+ final String nfPropertiesFilePath = System
+ .getProperty(NiFiProperties.PROPERTIES_FILE_PATH);
+ if (null == nfPropertiesFilePath || nfPropertiesFilePath.trim().length() == 0) {
+ throw new RuntimeException("Requires a system property called \'"
+ + NiFiProperties.PROPERTIES_FILE_PATH
+ + "\' and this is not set or has no value");
+ }
+ final File propertiesFile = new File(nfPropertiesFilePath);
+ if (!propertiesFile.exists()) {
+ throw new RuntimeException("Properties file doesn't exist \'"
+ + propertiesFile.getAbsolutePath() + "\'");
+ }
+ if (!propertiesFile.canRead()) {
+ throw new RuntimeException("Properties file exists but cannot be read \'"
+ + propertiesFile.getAbsolutePath() + "\'");
+ }
+ InputStream inStream = null;
+ try {
+ inStream = new BufferedInputStream(new FileInputStream(propertiesFile));
+ suspectInstance.load(inStream);
+ } catch (final Exception ex) {
+ LOG.error("Cannot load properties file due to " + ex.getLocalizedMessage());
+ throw new RuntimeException("Cannot load properties file due to "
+ + ex.getLocalizedMessage(), ex);
+ } finally {
+ if (null != inStream) {
+ try {
+ inStream.close();
+ } catch (final Exception ex) {
+ /**
+ * do nothing *
+ */
+ }
+ }
+ }
+ instance = suspectInstance;
+ }
+ return instance;
+ }
+
+ // getters for core properties //
+ public File getFlowConfigurationFile() {
+ try {
+ return new File(getProperty(FLOW_CONFIGURATION_FILE));
+ } catch (Exception ex) {
+ return null;
+ }
+ }
+
+ public File getFlowConfigurationFileDir() {
+ try {
+ return getFlowConfigurationFile().getParentFile();
+ } catch (Exception ex) {
+ return null;
+ }
+ }
+
+ private Integer getPropertyAsPort(final String propertyName, final Integer defaultValue) {
+ final String port = getProperty(propertyName);
+ if (StringUtils.isEmpty(port)) {
+ return defaultValue;
+ }
+ try {
+ final int val = Integer.parseInt(port);
+ if (val <= 0 || val > 65535) {
+ throw new RuntimeException("Valid port range is 0 - 65535 but got " + val);
+ }
+ return val;
+ } catch (final NumberFormatException e) {
+ return defaultValue;
+ }
+ }
+
+ public int getQueueSwapThreshold() {
+ final String thresholdValue = getProperty(QUEUE_SWAP_THRESHOLD);
+ if (thresholdValue == null) {
+ return DEFAULT_QUEUE_SWAP_THRESHOLD;
+ }
+
+ try {
+ return Integer.parseInt(thresholdValue);
+ } catch (final NumberFormatException e) {
+ return DEFAULT_QUEUE_SWAP_THRESHOLD;
+ }
+ }
+
+ public Integer getIntegerProperty(final String propertyName, final Integer defaultValue) {
+ final String value = getProperty(propertyName);
+ if (value == null) {
+ return defaultValue;
+ }
+
+ try {
+ return Integer.parseInt(getProperty(propertyName));
+ } catch (final Exception e) {
+ return defaultValue;
+ }
+ }
+
+ public int getSwapInThreads() {
+ return getIntegerProperty(SWAP_IN_THREADS, DEFAULT_SWAP_IN_THREADS);
+ }
+
+ public int getSwapOutThreads() {
+ final String value = getProperty(SWAP_OUT_THREADS);
+ if (value == null) {
+ return DEFAULT_SWAP_OUT_THREADS;
+ }
+
+ try {
+ return Integer.parseInt(getProperty(SWAP_OUT_THREADS));
+ } catch (final Exception e) {
+ return DEFAULT_SWAP_OUT_THREADS;
+ }
+ }
+
+ public String getSwapInPeriod() {
+ return getProperty(SWAP_IN_PERIOD, DEFAULT_SWAP_IN_PERIOD);
+ }
+
+ public String getSwapOutPeriod() {
+ return getProperty(SWAP_OUT_PERIOD, DEFAULT_SWAP_OUT_PERIOD);
+ }
+
+ public String getAdministrativeYieldDuration() {
+ return getProperty(ADMINISTRATIVE_YIELD_DURATION, DEFAULT_ADMINISTRATIVE_YIELD_DURATION);
+ }
+
+ /**
+ * The socket port to listen on for a Remote Input Port.
+ *
+ * @return the remote input port
+ */
+ public Integer getRemoteInputPort() {
+ return getPropertyAsPort(REMOTE_INPUT_PORT, DEFAULT_REMOTE_INPUT_PORT);
+ }
+
+ /**
+ * @return False if property value is 'false'; True otherwise.
+ */
+ public Boolean isSiteToSiteSecure() {
+ final String secureVal = getProperty(SITE_TO_SITE_SECURE, "true");
+
+ if ("false".equalsIgnoreCase(secureVal)) {
+ return false;
+ } else {
+ return true;
+ }
+
+ }
+
+ /**
+ * Returns the directory to which Templates are to be persisted
+ *
+ * @return the template directory
+ */
+ public Path getTemplateDirectory() {
+ final String strVal = getProperty(TEMPLATE_DIRECTORY);
+ return (strVal == null) ? DEFAULT_TEMPLATE_DIRECTORY : Paths.get(strVal);
+ }
+
+ /**
+ * Get the flow service write delay.
+ *
+ * @return The write delay
+ */
+ public String getFlowServiceWriteDelay() {
+ return getProperty(WRITE_DELAY_INTERVAL);
+ }
+
+ /**
+ * Returns whether the processors should be started automatically when the
+ * application loads.
+ *
+ * @return Whether to auto start the processors or not
+ */
+ public boolean getAutoResumeState() {
+ final String rawAutoResumeState = getProperty(AUTO_RESUME_STATE,
+ DEFAULT_AUTO_RESUME_STATE.toString());
+ return Boolean.parseBoolean(rawAutoResumeState);
+ }
+
+ /**
+ * Returns the number of partitions that should be used for the FlowFile
+ * Repository
+ *
+ * @return the number of partitions
+ */
+ public int getFlowFileRepositoryPartitions() {
+ final String rawProperty = getProperty(FLOWFILE_REPOSITORY_PARTITIONS,
+ DEFAULT_FLOWFILE_REPO_PARTITIONS);
+ return Integer.parseInt(rawProperty);
+ }
+
+ /**
+ * Returns the number of milliseconds between FlowFileRepository
+ * checkpointing
+ *
+ * @return the number of milliseconds between checkpoint events
+ */
+ public String getFlowFileRepositoryCheckpointInterval() {
+ return getProperty(FLOWFILE_REPOSITORY_CHECKPOINT_INTERVAL,
+ DEFAULT_FLOWFILE_CHECKPOINT_INTERVAL);
+ }
+
+ /**
+ * @return the restore directory or null if not configured
+ */
+ public File getRestoreDirectory() {
+ final String value = getProperty(RESTORE_DIRECTORY);
+ if (StringUtils.isBlank(value)) {
+ return null;
+ } else {
+ return new File(value);
+ }
+ }
+
+ /**
+ * @return the user authorities file
+ */
+ public File getAuthorityProviderConfiguraitonFile() {
+ final String value = getProperty(AUTHORITY_PROVIDER_CONFIGURATION_FILE);
+ if (StringUtils.isBlank(value)) {
+ return new File(DEFAULT_AUTHORITY_PROVIDER_CONFIGURATION_FILE);
+ } else {
+ return new File(value);
+ }
+ }
+
+ /**
+ * Will default to true unless the value is explicitly set to false.
+ *
+ * @return Whether client auth is required
+ */
+ public boolean getNeedClientAuth() {
+ boolean needClientAuth = true;
+ String rawNeedClientAuth = getProperty(SECURITY_NEED_CLIENT_AUTH);
+ if ("false".equalsIgnoreCase(rawNeedClientAuth)) {
+ needClientAuth = false;
+ }
+ return needClientAuth;
+ }
+
+ public String getUserCredentialCacheDuration() {
+ return getProperty(SECURITY_USER_CREDENTIAL_CACHE_DURATION,
+ DEFAULT_USER_CREDENTIAL_CACHE_DURATION);
+ }
+
+ public boolean getSupportNewAccountRequests() {
+ boolean shouldSupport = true;
+ String rawShouldSupport = getProperty(SECURITY_SUPPORT_NEW_ACCOUNT_REQUESTS);
+ if ("false".equalsIgnoreCase(rawShouldSupport)) {
+ shouldSupport = false;
+ }
+ return shouldSupport;
+ }
+
+ // getters for web properties //
+ public Integer getPort() {
+ Integer port = null;
+ try {
+ port = Integer.parseInt(getProperty(WEB_HTTP_PORT));
+ } catch (NumberFormatException nfe) {
+ }
+ return port;
+ }
+
+ public Integer getSslPort() {
+ Integer sslPort = null;
+ try {
+ sslPort = Integer.parseInt(getProperty(WEB_HTTPS_PORT));
+ } catch (NumberFormatException nfe) {
+ }
+ return sslPort;
+ }
+
+ public int getWebThreads() {
+ return getIntegerProperty(WEB_THREADS, DEFAULT_WEB_THREADS);
+ }
+
+ public File getWebWorkingDirectory() {
+ return new File(getProperty(WEB_WORKING_DIR, DEFAULT_WEB_WORKING_DIR));
+ }
+
+ public File getComponentDocumentationWorkingDirectory() {
+ return new File(getProperty(COMPONENT_DOCS_DIRECTORY, DEFAULT_COMPONENT_DOCS_DIRECTORY));
+ }
+
+ public File getNarWorkingDirectory() {
+ return new File(getProperty(NAR_WORKING_DIRECTORY, DEFAULT_NAR_WORKING_DIR));
+ }
+
+ public File getFrameworkWorkingDirectory() {
+ return new File(getNarWorkingDirectory(), "framework");
+ }
+
+ public File getExtensionsWorkingDirectory() {
+ return new File(getNarWorkingDirectory(), "extensions");
+ }
+
+ public List<Path> getNarLibraryDirectories() {
+
+ List<Path> narLibraryPaths = new ArrayList<>();
+
+ // go through each property
+ for (String propertyName : stringPropertyNames()) {
+ // determine if the property is a nar library path
+ if (StringUtils.startsWith(propertyName, NAR_LIBRARY_DIRECTORY_PREFIX)
+ || NAR_LIBRARY_DIRECTORY.equals(propertyName)) {
+ // attempt to resolve the path specified
+ String narLib = getProperty(propertyName);
+ if (!StringUtils.isBlank(narLib)) {
+ narLibraryPaths.add(Paths.get(narLib));
+ }
+ }
+ }
+
+ if (narLibraryPaths.isEmpty()) {
+ narLibraryPaths.add(Paths.get(DEFAULT_NAR_LIBRARY_DIR));
+ }
+
+ return narLibraryPaths;
+ }
+
+ // getters for ui properties //
+ /**
+ * Get the title for the UI.
+ *
+ * @return The UI title
+ */
+ public String getUiTitle() {
+ return this.getProperty(VERSION, DEFAULT_TITLE);
+ }
+
+ /**
+ * Get the banner text.
+ *
+ * @return The banner text
+ */
+ public String getBannerText() {
+ return this.getProperty(UI_BANNER_TEXT, StringUtils.EMPTY);
+ }
+
+ /**
+ * Returns the auto refresh interval in seconds.
+ *
+ * @return the interval over which the properties should auto refresh
+ */
+ public String getAutoRefreshInterval() {
+ return getProperty(UI_AUTO_REFRESH_INTERVAL);
+ }
+
+ // getters for cluster protocol properties //
+ public String getClusterProtocolHeartbeatInterval() {
+ return getProperty(CLUSTER_PROTOCOL_HEARTBEAT_INTERVAL,
+ DEFAULT_CLUSTER_PROTOCOL_HEARTBEAT_INTERVAL);
+ }
+
+ public String getNodeHeartbeatInterval() {
+ return getClusterProtocolHeartbeatInterval();
+ }
+
+ public String getClusterProtocolSocketTimeout() {
+ return getProperty(CLUSTER_PROTOCOL_SOCKET_TIMEOUT, DEFAULT_CLUSTER_PROTOCOL_SOCKET_TIMEOUT);
+ }
+
+ public String getClusterProtocolConnectionHandshakeTimeout() {
+ return getProperty(CLUSTER_PROTOCOL_CONNECTION_HANDSHAKE_TIMEOUT,
+ DEFAULT_CLUSTER_PROTOCOL_CONNECTION_HANDSHAKE_TIMEOUT);
+ }
+
+ public boolean getClusterProtocolUseMulticast() {
+ return Boolean.parseBoolean(getProperty(CLUSTER_PROTOCOL_USE_MULTICAST));
+ }
+
+ public InetSocketAddress getClusterProtocolMulticastAddress() {
+ try {
+ String multicastAddress = getProperty(CLUSTER_PROTOCOL_MULTICAST_ADDRESS);
+ int multicastPort = Integer.parseInt(getProperty(CLUSTER_PROTOCOL_MULTICAST_PORT));
+ return new InetSocketAddress(multicastAddress, multicastPort);
+ } catch (Exception ex) {
+ throw new RuntimeException("Invalid multicast address/port due to: " + ex, ex);
+ }
+ }
+
+ public String getClusterProtocolMulticastServiceBroadcastDelay() {
+ return getProperty(CLUSTER_PROTOCOL_MULTICAST_SERVICE_BROADCAST_DELAY);
+ }
+
+ public File getPersistentStateDirectory() {
+ final String dirName = getProperty(PERSISTENT_STATE_DIRECTORY,
+ DEFAULT_PERSISTENT_STATE_DIRECTORY);
+ final File file = new File(dirName);
+ if (!file.exists()) {
+ file.mkdirs();
+ }
+ return file;
+ }
+
+ public int getClusterProtocolMulticastServiceLocatorAttempts() {
+ try {
+ return Integer
+ .parseInt(getProperty(CLUSTER_PROTOCOL_MULTICAST_SERVICE_LOCATOR_ATTEMPTS));
+ } catch (NumberFormatException nfe) {
+ return DEFAULT_CLUSTER_PROTOCOL_MULTICAST_SERVICE_LOCATOR_ATTEMPTS;
+ }
+ }
+
+ public String getClusterProtocolMulticastServiceLocatorAttemptsDelay() {
+ return getProperty(CLUSTER_PROTOCOL_MULTICAST_SERVICE_LOCATOR_ATTEMPTS_DELAY,
+ DEFAULT_CLUSTER_PROTOCOL_MULTICAST_SERVICE_LOCATOR_ATTEMPTS_DELAY);
+ }
+
+ // getters for cluster node properties //
+ public boolean isNode() {
+ return Boolean.parseBoolean(getProperty(CLUSTER_IS_NODE));
+ }
+
+ public InetSocketAddress getClusterNodeProtocolAddress() {
+ try {
+ String socketAddress = getProperty(CLUSTER_NODE_ADDRESS);
+ if (StringUtils.isBlank(socketAddress)) {
+ socketAddress = "localhost";
+ }
+ int socketPort = getClusterNodeProtocolPort();
+ return InetSocketAddress.createUnresolved(socketAddress, socketPort);
+ } catch (Exception ex) {
+ throw new RuntimeException("Invalid node protocol address/port due to: " + ex, ex);
+ }
+ }
+
+ public Integer getClusterNodeProtocolPort() {
+ try {
+ return Integer.parseInt(getProperty(CLUSTER_NODE_PROTOCOL_PORT));
+ } catch (NumberFormatException nfe) {
+ return null;
+ }
+ }
+
+ public int getClusterNodeProtocolThreads() {
+ try {
+ return Integer.parseInt(getProperty(CLUSTER_NODE_PROTOCOL_THREADS));
+ } catch (NumberFormatException nfe) {
+ return DEFAULT_CLUSTER_NODE_PROTOCOL_THREADS;
+ }
+ }
+
+ public InetSocketAddress getClusterNodeUnicastManagerProtocolAddress() {
+ try {
+ String socketAddress = getProperty(CLUSTER_NODE_UNICAST_MANAGER_ADDRESS);
+ if (StringUtils.isBlank(socketAddress)) {
+ socketAddress = "localhost";
+ }
+ int socketPort = Integer
+ .parseInt(getProperty(CLUSTER_NODE_UNICAST_MANAGER_PROTOCOL_PORT));
+ return InetSocketAddress.createUnresolved(socketAddress, socketPort);
+ } catch (Exception ex) {
+ throw new RuntimeException("Invalid unicast manager address/port due to: " + ex, ex);
+ }
+ }
+
+ // getters for cluster manager properties //
+ public boolean isClusterManager() {
+ return Boolean.parseBoolean(getProperty(CLUSTER_IS_MANAGER));
+ }
+
+ public InetSocketAddress getClusterManagerProtocolAddress() {
+ try {
+ String socketAddress = getProperty(CLUSTER_MANAGER_ADDRESS);
+ if (StringUtils.isBlank(socketAddress)) {
+ socketAddress = "localhost";
+ }
+ int socketPort = getClusterManagerProtocolPort();
+ return InetSocketAddress.createUnresolved(socketAddress, socketPort);
+ } catch (Exception ex) {
+ throw new RuntimeException("Invalid manager protocol address/port due to: " + ex, ex);
+ }
+ }
+
+ public Integer getClusterManagerProtocolPort() {
+ try {
+ return Integer.parseInt(getProperty(CLUSTER_MANAGER_PROTOCOL_PORT));
+ } catch (NumberFormatException nfe) {
+ return null;
+ }
+ }
+
+ public File getClusterManagerNodeFirewallFile() {
+ final String firewallFile = getProperty(CLUSTER_MANAGER_NODE_FIREWALL_FILE);
+ if (StringUtils.isBlank(firewallFile)) {
+ return null;
+ } else {
+ return new File(firewallFile);
+ }
+ }
+
+ public int getClusterManagerNodeEventHistorySize() {
+ try {
+ return Integer.parseInt(getProperty(CLUSTER_MANAGER_NODE_EVENT_HISTORY_SIZE));
+ } catch (NumberFormatException nfe) {
+ return DEFAULT_CLUSTER_MANAGER_NODE_EVENT_HISTORY_SIZE;
+ }
+ }
+
+ public String getClusterManagerNodeApiConnectionTimeout() {
+ return getProperty(CLUSTER_MANAGER_NODE_API_CONNECTION_TIMEOUT,
+ DEFAULT_CLUSTER_MANAGER_NODE_API_CONNECTION_TIMEOUT);
+ }
+
+ public String getClusterManagerNodeApiReadTimeout() {
+ return getProperty(CLUSTER_MANAGER_NODE_API_READ_TIMEOUT,
+ DEFAULT_CLUSTER_MANAGER_NODE_API_READ_TIMEOUT);
+ }
+
+ public int getClusterManagerNodeApiRequestThreads() {
+ try {
+ return Integer.parseInt(getProperty(CLUSTER_MANAGER_NODE_API_REQUEST_THREADS));
+ } catch (NumberFormatException nfe) {
+ return DEFAULT_CLUSTER_MANAGER_NODE_API_NUM_REQUEST_THREADS;
+ }
+ }
+
+ public String getClusterManagerFlowRetrievalDelay() {
+ return getProperty(CLUSTER_MANAGER_FLOW_RETRIEVAL_DELAY,
+ DEFAULT_CLUSTER_MANAGER_FLOW_RETRIEVAL_DELAY);
+ }
+
+ public int getClusterManagerProtocolThreads() {
+ try {
+ return Integer.parseInt(getProperty(CLUSTER_MANAGER_PROTOCOL_THREADS));
+ } catch (NumberFormatException nfe) {
+ return DEFAULT_CLUSTER_MANAGER_PROTOCOL_THREADS;
+ }
+ }
+
+ public String getClusterManagerSafeModeDuration() {
+ return getProperty(CLUSTER_MANAGER_SAFEMODE_DURATION,
+ DEFAULT_CLUSTER_MANAGER_SAFEMODE_DURATION);
+ }
+
+ public String getClusterProtocolManagerToNodeApiScheme() {
+ final String isSecureProperty = getProperty(CLUSTER_PROTOCOL_IS_SECURE);
+ if (Boolean.valueOf(isSecureProperty)) {
+ return "https";
+ } else {
+ return "http";
+ }
+ }
+
+ public InetSocketAddress getNodeApiAddress() {
+
+ final String rawScheme = getClusterProtocolManagerToNodeApiScheme();
+ final String scheme = (rawScheme == null) ? "http" : rawScheme;
+
+ final String host;
+ final Integer port;
+ if ("http".equalsIgnoreCase(scheme)) {
+ // get host
+ if (StringUtils.isBlank(getProperty(WEB_HTTP_HOST))) {
+ host = "localhost";
+ } else {
+ host = getProperty(WEB_HTTP_HOST);
+ }
+ // get port
+ port = getPort();
+
+ if (port == null) {
+ throw new RuntimeException(String.format("The %s must be specified if running in a cluster with %s set to false.", WEB_HTTP_PORT, CLUSTER_PROTOCOL_IS_SECURE));
+ }
+ } else {
+ // get host
+ if (StringUtils.isBlank(getProperty(WEB_HTTPS_HOST))) {
+ host = "localhost";
+ } else {
+ host = getProperty(WEB_HTTPS_HOST);
+ }
+ // get port
+ port = getSslPort();
+
+ if (port == null) {
+ throw new RuntimeException(String.format("The %s must be specified if running in a cluster with %s set to true.", WEB_HTTPS_PORT, CLUSTER_PROTOCOL_IS_SECURE));
+ }
+ }
+
+ return InetSocketAddress.createUnresolved(host, port);
+
+ }
+
+ /**
+ * Returns the database repository path. It simply returns the value
+ * configured. No directories will be created as a result of this operation.
+ *
+ * @return database repository path
+ * @throws InvalidPathException If the configured path is invalid
+ */
+ public Path getDatabaseRepositoryPath() {
+ return Paths.get(getProperty(REPOSITORY_DATABASE_DIRECTORY));
+ }
+
+ /**
+ * Returns the flow file repository path. It simply returns the value
+ * configured. No directories will be created as a result of this operation.
+ *
+ * @return database repository path
+ * @throws InvalidPathException If the configured path is invalid
+ */
+ public Path getFlowFileRepositoryPath() {
+ return Paths.get(getProperty(FLOWFILE_REPOSITORY_DIRECTORY));
+ }
+
+ /**
+ * Returns the content repository paths. This method returns a mapping of
+ * file repository name to file repository paths. It simply returns the
+ * values configured. No directories will be created as a result of this
+ * operation.
+ *
+ * @return file repositories paths
+ * @throws InvalidPathException If any of the configured paths are invalid
+ */
+ public Map<String, Path> getContentRepositoryPaths() {
+ final Map<String, Path> contentRepositoryPaths = new HashMap<>();
+
+ // go through each property
+ for (String propertyName : stringPropertyNames()) {
+ // determine if the property is a file repository path
+ if (StringUtils.startsWith(propertyName, REPOSITORY_CONTENT_PREFIX)) {
+ // get the repository key
+ final String key = StringUtils.substringAfter(propertyName,
+ REPOSITORY_CONTENT_PREFIX);
+
+ // attempt to resolve the path specified
+ contentRepositoryPaths.put(key, Paths.get(getProperty(propertyName)));
+ }
+ }
+ return contentRepositoryPaths;
+ }
+
+ /**
+ * Returns the provenance repository paths. This method returns a mapping of
+ * file repository name to file repository paths. It simply returns the
+ * values configured. No directories will be created as a result of this
+ * operation.
+ *
+ * @return the name and paths of all provenance repository locations
+ */
+ public Map<String, Path> getProvenanceRepositoryPaths() {
+ final Map<String, Path> provenanceRepositoryPaths = new HashMap<>();
+
+ // go through each property
+ for (String propertyName : stringPropertyNames()) {
+ // determine if the property is a file repository path
+ if (StringUtils.startsWith(propertyName, PROVENANCE_REPO_DIRECTORY_PREFIX)) {
+ // get the repository key
+ final String key = StringUtils.substringAfter(propertyName,
+ PROVENANCE_REPO_DIRECTORY_PREFIX);
+
+ // attempt to resolve the path specified
+ provenanceRepositoryPaths.put(key, Paths.get(getProperty(propertyName)));
+ }
+ }
+ return provenanceRepositoryPaths;
+ }
+
+ public int getMaxFlowFilesPerClaim() {
+ try {
+ return Integer.parseInt(getProperty(MAX_FLOWFILES_PER_CLAIM));
+ } catch (NumberFormatException nfe) {
+ return DEFAULT_MAX_FLOWFILES_PER_CLAIM;
+ }
+ }
+
+ public String getMaxAppendableClaimSize() {
+ return getProperty(MAX_APPENDABLE_CLAIM_SIZE);
+ }
+
+ @Override
+ public String getProperty(final String key, final String defaultValue) {
+ final String value = super.getProperty(key, defaultValue);
+ if (value == null) {
+ return null;
+ }
+
+ if (value.trim().isEmpty()) {
+ return defaultValue;
+ }
+ return value;
+ }
+
+ public String getBoredYieldDuration() {
+ return getProperty(BORED_YIELD_DURATION, DEFAULT_BORED_YIELD_DURATION);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/StringUtils.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/StringUtils.java b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/StringUtils.java
new file mode 100644
index 0000000..aa6f8f3
--- /dev/null
+++ b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/StringUtils.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+/**
+ * String Utils based on the Apache Commons Lang String Utils.
+ * These simple util methods here allow us to avoid a dependency in the core
+ */
+public class StringUtils {
+
+ public static final String EMPTY = "";
+
+ public static boolean isBlank(final String str) {
+ if (str == null || str.isEmpty()) {
+ return true;
+ }
+ for (int i = 0; i < str.length(); i++) {
+ if (!Character.isWhitespace(str.charAt(i))) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ public static boolean isEmpty(final String str) {
+ return str == null || str.isEmpty();
+ }
+
+ public static boolean startsWith(final String str, final String prefix) {
+ if (str == null || prefix == null) {
+ return (str == null && prefix == null);
+ }
+ if (prefix.length() > str.length()) {
+ return false;
+ }
+ return str.regionMatches(false, 0, prefix, 0, prefix.length());
+ }
+
+ public static String substringAfter(final String str, final String separator) {
+ if (isEmpty(str)) {
+ return str;
+ }
+ if (separator == null) {
+ return EMPTY;
+ }
+ int pos = str.indexOf(separator);
+ if (pos == -1) {
+ return EMPTY;
+ }
+ return str.substring(pos + separator.length());
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-properties/src/test/java/org/apache/nifi/util/NiFiPropertiesTest.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-properties/src/test/java/org/apache/nifi/util/NiFiPropertiesTest.java b/nifi-commons/nifi-properties/src/test/java/org/apache/nifi/util/NiFiPropertiesTest.java
new file mode 100644
index 0000000..9c1c8d7
--- /dev/null
+++ b/nifi-commons/nifi-properties/src/test/java/org/apache/nifi/util/NiFiPropertiesTest.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.net.URISyntaxException;
+import java.nio.file.Path;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+
+public class NiFiPropertiesTest {
+
+ @Test
+ public void testProperties() {
+
+ NiFiProperties properties = loadSpecifiedProperties("/NiFiProperties/conf/nifi.properties");
+
+ assertEquals("UI Banner Text", properties.getBannerText());
+
+ Set<File> expectedDirectories = new HashSet<>();
+ expectedDirectories.add(new File("./target/resources/NiFiProperties/lib/"));
+ expectedDirectories.add(new File("./target/resources/NiFiProperties/lib2/"));
+
+ Set<String> directories = new HashSet<>();
+ for (Path narLibDir : properties.getNarLibraryDirectories()) {
+ directories.add(narLibDir.toString());
+ }
+
+ Assert.assertEquals("Did not have the anticipated number of directories", expectedDirectories.size(), directories.size());
+ for (File expectedDirectory : expectedDirectories) {
+ Assert.assertTrue("Listed directories did not contain expected directory", directories.contains(expectedDirectory.getPath()));
+ }
+ }
+
+ @Test
+ public void testMissingProperties() {
+
+ NiFiProperties properties = loadSpecifiedProperties("/NiFiProperties/conf/nifi.missing.properties");
+
+ List<Path> directories = properties.getNarLibraryDirectories();
+
+ assertEquals(1, directories.size());
+
+ assertEquals(new File(NiFiProperties.DEFAULT_NAR_LIBRARY_DIR).getPath(), directories.get(0)
+ .toString());
+
+ }
+
+ @Test
+ public void testBlankProperties() {
+
+ NiFiProperties properties = loadSpecifiedProperties("/NiFiProperties/conf/nifi.blank.properties");
+
+ List<Path> directories = properties.getNarLibraryDirectories();
+
+ assertEquals(1, directories.size());
+
+ assertEquals(new File(NiFiProperties.DEFAULT_NAR_LIBRARY_DIR).getPath(), directories.get(0)
+ .toString());
+
+ }
+
+ private NiFiProperties loadSpecifiedProperties(String propertiesFile) {
+
+ String filePath;
+ try {
+ filePath = NiFiPropertiesTest.class.getResource(propertiesFile).toURI().getPath();
+ } catch (URISyntaxException ex) {
+ throw new RuntimeException("Cannot load properties file due to "
+ + ex.getLocalizedMessage(), ex);
+ }
+
+ System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, filePath);
+
+ NiFiProperties properties = NiFiProperties.getInstance();
+
+ // clear out existing properties
+ for (String prop : properties.stringPropertyNames()) {
+ properties.remove(prop);
+ }
+
+ InputStream inStream = null;
+ try {
+ inStream = new BufferedInputStream(new FileInputStream(filePath));
+ properties.load(inStream);
+ } catch (final Exception ex) {
+ throw new RuntimeException("Cannot load properties file due to "
+ + ex.getLocalizedMessage(), ex);
+ } finally {
+ if (null != inStream) {
+ try {
+ inStream.close();
+ } catch (final Exception ex) {
+ /**
+ * do nothing *
+ */
+ }
+ }
+ }
+
+ return properties;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.blank.properties
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.blank.properties b/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.blank.properties
new file mode 100644
index 0000000..1f853b9
--- /dev/null
+++ b/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.blank.properties
@@ -0,0 +1,128 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Core Properties #
+nifi.version=nifi-test 3.0.0
+nifi.flow.configuration.file=./target/flow.xml.gz
+nifi.flow.configuration.archive.dir=./target/archive/
+nifi.flowcontroller.autoResumeState=true
+nifi.flowcontroller.graceful.shutdown.period=10 sec
+nifi.flowservice.writedelay.interval=2 sec
+nifi.administrative.yield.duration=30 sec
+
+nifi.reporting.task.configuration.file=./target/reporting-tasks.xml
+nifi.controller.service.configuration.file=./target/controller-services.xml
+nifi.templates.directory=./target/templates
+nifi.ui.banner.text=UI Banner Text
+nifi.ui.autorefresh.interval=30 sec
+nifi.nar.library.directory=
+nifi.custom.nar.library.directory.alt=
+nifi.nar.working.directory=./target/work/nar/
+
+# H2 Settings
+nifi.database.directory=./target/database_repository
+nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
+
+# FlowFile Repository
+nifi.flowfile.repository.directory=./target/test-repo
+nifi.flowfile.repository.partitions=1
+nifi.flowfile.repository.checkpoint.interval=2 mins
+nifi.queue.swap.threshold=20000
+nifi.swap.storage.directory=./target/test-repo/swap
+nifi.swap.in.period=5 sec
+nifi.swap.in.threads=1
+nifi.swap.out.period=5 sec
+nifi.swap.out.threads=4
+
+# Content Repository
+nifi.content.claim.max.appendable.size=10 MB
+nifi.content.claim.max.flow.files=100
+nifi.content.repository.directory.default=./target/content_repository
+
+# Provenance Repository Properties
+nifi.provenance.repository.storage.directory=./target/provenance_repository
+nifi.provenance.repository.max.storage.time=24 hours
+nifi.provenance.repository.max.storage.size=1 GB
+nifi.provenance.repository.rollover.time=5 mins
+nifi.provenance.repository.rollover.size=100 MB
+
+# Site to Site properties
+nifi.remote.input.socket.port=9990
+nifi.remote.input.secure=true
+
+# web properties #
+nifi.web.war.directory=./target/lib
+nifi.web.http.host=
+nifi.web.http.port=8080
+nifi.web.https.host=
+nifi.web.https.port=
+nifi.web.jetty.working.directory=./target/work/jetty
+
+# security properties #
+nifi.sensitive.props.key=key
+nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
+nifi.sensitive.props.provider=BC
+
+nifi.security.keystore=
+nifi.security.keystoreType=
+nifi.security.keystorePasswd=
+nifi.security.keyPasswd=
+nifi.security.truststore=
+nifi.security.truststoreType=
+nifi.security.truststorePasswd=
+nifi.security.needClientAuth=
+nifi.security.authorizedUsers.file=./target/conf/authorized-users.xml
+nifi.security.user.credential.cache.duration=24 hours
+nifi.security.user.authority.provider=nifi.authorization.FileAuthorizationProvider
+nifi.security.support.new.account.requests=
+nifi.security.default.user.roles=
+
+# cluster common properties (cluster manager and nodes must have same values) #
+nifi.cluster.protocol.heartbeat.interval=5 sec
+nifi.cluster.protocol.is.secure=false
+nifi.cluster.protocol.socket.timeout=30 sec
+nifi.cluster.protocol.connection.handshake.timeout=45 sec
+# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured #
+nifi.cluster.protocol.use.multicast=false
+nifi.cluster.protocol.multicast.address=
+nifi.cluster.protocol.multicast.port=
+nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms
+nifi.cluster.protocol.multicast.service.locator.attempts=3
+nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec
+
+# cluster node properties (only configure for cluster nodes) #
+nifi.cluster.is.node=false
+nifi.cluster.node.address=
+nifi.cluster.node.protocol.port=
+nifi.cluster.node.protocol.threads=2
+# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx #
+nifi.cluster.node.unicast.manager.address=
+nifi.cluster.node.unicast.manager.protocol.port=
+nifi.cluster.node.unicast.manager.authority.provider.port=
+
+# cluster manager properties (only configure for cluster manager) #
+nifi.cluster.is.manager=false
+nifi.cluster.manager.address=
+nifi.cluster.manager.protocol.port=
+nifi.cluster.manager.authority.provider.port=
+nifi.cluster.manager.authority.provider.threads=10
+nifi.cluster.manager.node.firewall.file=
+nifi.cluster.manager.node.event.history.size=10
+nifi.cluster.manager.node.api.connection.timeout=30 sec
+nifi.cluster.manager.node.api.read.timeout=30 sec
+nifi.cluster.manager.node.api.request.threads=10
+nifi.cluster.manager.flow.retrieval.delay=5 sec
+nifi.cluster.manager.protocol.threads=10
+nifi.cluster.manager.safemode.duration=0 sec
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.missing.properties
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.missing.properties b/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.missing.properties
new file mode 100644
index 0000000..dbb8978
--- /dev/null
+++ b/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.missing.properties
@@ -0,0 +1,126 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Core Properties #
+nifi.version=nifi-test 3.0.0
+nifi.flow.configuration.file=./target/flow.xml.gz
+nifi.flow.configuration.archive.dir=./target/archive/
+nifi.flowcontroller.autoResumeState=true
+nifi.flowcontroller.graceful.shutdown.period=10 sec
+nifi.flowservice.writedelay.interval=2 sec
+nifi.administrative.yield.duration=30 sec
+
+nifi.reporting.task.configuration.file=./target/reporting-tasks.xml
+nifi.controller.service.configuration.file=./target/controller-services.xml
+nifi.templates.directory=./target/templates
+nifi.ui.banner.text=UI Banner Text
+nifi.ui.autorefresh.interval=30 sec
+nifi.nar.working.directory=./target/work/nar/
+
+# H2 Settings
+nifi.database.directory=./target/database_repository
+nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
+
+# FlowFile Repository
+nifi.flowfile.repository.directory=./target/test-repo
+nifi.flowfile.repository.partitions=1
+nifi.flowfile.repository.checkpoint.interval=2 mins
+nifi.queue.swap.threshold=20000
+nifi.swap.storage.directory=./target/test-repo/swap
+nifi.swap.in.period=5 sec
+nifi.swap.in.threads=1
+nifi.swap.out.period=5 sec
+nifi.swap.out.threads=4
+
+# Content Repository
+nifi.content.claim.max.appendable.size=10 MB
+nifi.content.claim.max.flow.files=100
+nifi.content.repository.directory.default=./target/content_repository
+
+# Provenance Repository Properties
+nifi.provenance.repository.storage.directory=./target/provenance_repository
+nifi.provenance.repository.max.storage.time=24 hours
+nifi.provenance.repository.max.storage.size=1 GB
+nifi.provenance.repository.rollover.time=5 mins
+nifi.provenance.repository.rollover.size=100 MB
+
+# Site to Site properties
+nifi.remote.input.socket.port=9990
+nifi.remote.input.secure=true
+
+# web properties #
+nifi.web.war.directory=./target/lib
+nifi.web.http.host=
+nifi.web.http.port=8080
+nifi.web.https.host=
+nifi.web.https.port=
+nifi.web.jetty.working.directory=./target/work/jetty
+
+# security properties #
+nifi.sensitive.props.key=key
+nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
+nifi.sensitive.props.provider=BC
+
+nifi.security.keystore=
+nifi.security.keystoreType=
+nifi.security.keystorePasswd=
+nifi.security.keyPasswd=
+nifi.security.truststore=
+nifi.security.truststoreType=
+nifi.security.truststorePasswd=
+nifi.security.needClientAuth=
+nifi.security.authorizedUsers.file=./target/conf/authorized-users.xml
+nifi.security.user.credential.cache.duration=24 hours
+nifi.security.user.authority.provider=nifi.authorization.FileAuthorizationProvider
+nifi.security.support.new.account.requests=
+nifi.security.default.user.roles=
+
+# cluster common properties (cluster manager and nodes must have same values) #
+nifi.cluster.protocol.heartbeat.interval=5 sec
+nifi.cluster.protocol.is.secure=false
+nifi.cluster.protocol.socket.timeout=30 sec
+nifi.cluster.protocol.connection.handshake.timeout=45 sec
+# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured #
+nifi.cluster.protocol.use.multicast=false
+nifi.cluster.protocol.multicast.address=
+nifi.cluster.protocol.multicast.port=
+nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms
+nifi.cluster.protocol.multicast.service.locator.attempts=3
+nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec
+
+# cluster node properties (only configure for cluster nodes) #
+nifi.cluster.is.node=false
+nifi.cluster.node.address=
+nifi.cluster.node.protocol.port=
+nifi.cluster.node.protocol.threads=2
+# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx #
+nifi.cluster.node.unicast.manager.address=
+nifi.cluster.node.unicast.manager.protocol.port=
+nifi.cluster.node.unicast.manager.authority.provider.port=
+
+# cluster manager properties (only configure for cluster manager) #
+nifi.cluster.is.manager=false
+nifi.cluster.manager.address=
+nifi.cluster.manager.protocol.port=
+nifi.cluster.manager.authority.provider.port=
+nifi.cluster.manager.authority.provider.threads=10
+nifi.cluster.manager.node.firewall.file=
+nifi.cluster.manager.node.event.history.size=10
+nifi.cluster.manager.node.api.connection.timeout=30 sec
+nifi.cluster.manager.node.api.read.timeout=30 sec
+nifi.cluster.manager.node.api.request.threads=10
+nifi.cluster.manager.flow.retrieval.delay=5 sec
+nifi.cluster.manager.protocol.threads=10
+nifi.cluster.manager.safemode.duration=0 sec
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.properties
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.properties b/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.properties
new file mode 100644
index 0000000..ed44b88
--- /dev/null
+++ b/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.properties
@@ -0,0 +1,128 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Core Properties #
+nifi.version=nifi-test 3.0.0
+nifi.flow.configuration.file=./target/flow.xml.gz
+nifi.flow.configuration.archive.dir=./target/archive/
+nifi.flowcontroller.autoResumeState=true
+nifi.flowcontroller.graceful.shutdown.period=10 sec
+nifi.flowservice.writedelay.interval=2 sec
+nifi.administrative.yield.duration=30 sec
+
+nifi.reporting.task.configuration.file=./target/reporting-tasks.xml
+nifi.controller.service.configuration.file=./target/controller-services.xml
+nifi.templates.directory=./target/templates
+nifi.ui.banner.text=UI Banner Text
+nifi.ui.autorefresh.interval=30 sec
+nifi.nar.library.directory=./target/resources/NiFiProperties/lib/
+nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/
+nifi.nar.working.directory=./target/work/nar/
+
+# H2 Settings
+nifi.database.directory=./target/database_repository
+nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
+
+# FlowFile Repository
+nifi.flowfile.repository.directory=./target/test-repo
+nifi.flowfile.repository.partitions=1
+nifi.flowfile.repository.checkpoint.interval=2 mins
+nifi.queue.swap.threshold=20000
+nifi.swap.storage.directory=./target/test-repo/swap
+nifi.swap.in.period=5 sec
+nifi.swap.in.threads=1
+nifi.swap.out.period=5 sec
+nifi.swap.out.threads=4
+
+# Content Repository
+nifi.content.claim.max.appendable.size=10 MB
+nifi.content.claim.max.flow.files=100
+nifi.content.repository.directory.default=./target/content_repository
+
+# Provenance Repository Properties
+nifi.provenance.repository.storage.directory=./target/provenance_repository
+nifi.provenance.repository.max.storage.time=24 hours
+nifi.provenance.repository.max.storage.size=1 GB
+nifi.provenance.repository.rollover.time=5 mins
+nifi.provenance.repository.rollover.size=100 MB
+
+# Site to Site properties
+nifi.remote.input.socket.port=9990
+nifi.remote.input.secure=true
+
+# web properties #
+nifi.web.war.directory=./target/lib
+nifi.web.http.host=
+nifi.web.http.port=8080
+nifi.web.https.host=
+nifi.web.https.port=
+nifi.web.jetty.working.directory=./target/work/jetty
+
+# security properties #
+nifi.sensitive.props.key=key
+nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
+nifi.sensitive.props.provider=BC
+
+nifi.security.keystore=
+nifi.security.keystoreType=
+nifi.security.keystorePasswd=
+nifi.security.keyPasswd=
+nifi.security.truststore=
+nifi.security.truststoreType=
+nifi.security.truststorePasswd=
+nifi.security.needClientAuth=
+nifi.security.authorizedUsers.file=./target/conf/authorized-users.xml
+nifi.security.user.credential.cache.duration=24 hours
+nifi.security.user.authority.provider=nifi.authorization.FileAuthorizationProvider
+nifi.security.support.new.account.requests=
+nifi.security.default.user.roles=
+
+# cluster common properties (cluster manager and nodes must have same values) #
+nifi.cluster.protocol.heartbeat.interval=5 sec
+nifi.cluster.protocol.is.secure=false
+nifi.cluster.protocol.socket.timeout=30 sec
+nifi.cluster.protocol.connection.handshake.timeout=45 sec
+# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured #
+nifi.cluster.protocol.use.multicast=false
+nifi.cluster.protocol.multicast.address=
+nifi.cluster.protocol.multicast.port=
+nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms
+nifi.cluster.protocol.multicast.service.locator.attempts=3
+nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec
+
+# cluster node properties (only configure for cluster nodes) #
+nifi.cluster.is.node=false
+nifi.cluster.node.address=
+nifi.cluster.node.protocol.port=
+nifi.cluster.node.protocol.threads=2
+# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx #
+nifi.cluster.node.unicast.manager.address=
+nifi.cluster.node.unicast.manager.protocol.port=
+nifi.cluster.node.unicast.manager.authority.provider.port=
+
+# cluster manager properties (only configure for cluster manager) #
+nifi.cluster.is.manager=false
+nifi.cluster.manager.address=
+nifi.cluster.manager.protocol.port=
+nifi.cluster.manager.authority.provider.port=
+nifi.cluster.manager.authority.provider.threads=10
+nifi.cluster.manager.node.firewall.file=
+nifi.cluster.manager.node.event.history.size=10
+nifi.cluster.manager.node.api.connection.timeout=30 sec
+nifi.cluster.manager.node.api.read.timeout=30 sec
+nifi.cluster.manager.node.api.request.threads=10
+nifi.cluster.manager.flow.retrieval.delay=5 sec
+nifi.cluster.manager.protocol.threads=10
+nifi.cluster.manager.safemode.duration=0 sec
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-security-utils/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-security-utils/pom.xml b/nifi-commons/nifi-security-utils/pom.xml
new file mode 100644
index 0000000..4e222e0
--- /dev/null
+++ b/nifi-commons/nifi-security-utils/pom.xml
@@ -0,0 +1,35 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-commons</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>nifi-security-utils</artifactId>
+ <description>Contains security functionality.</description>
+ <dependencies>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
+ </dependencies>
+</project>
+
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/CertificateUtils.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/CertificateUtils.java b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/CertificateUtils.java
new file mode 100644
index 0000000..5126933
--- /dev/null
+++ b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/CertificateUtils.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.security.util;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.net.URL;
+import java.security.KeyStore;
+import java.security.cert.CertificateParsingException;
+import java.security.cert.X509Certificate;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public final class CertificateUtils {
+
+ private static final Logger logger = LoggerFactory.getLogger(CertificateUtils.class);
+
+ /**
+ * Returns true if the given keystore can be loaded using the given keystore
+ * type and password. Returns false otherwise.
+ *
+ * @param keystore the keystore to validate
+ * @param keystoreType the type of the keystore
+ * @param password the password to access the keystore
+ * @return true if valid; false otherwise
+ */
+ public static boolean isStoreValid(final URL keystore, final KeystoreType keystoreType, final char[] password) {
+
+ if (keystore == null) {
+ throw new IllegalArgumentException("keystore may not be null");
+ } else if (keystoreType == null) {
+ throw new IllegalArgumentException("keystore type may not be null");
+ } else if (password == null) {
+ throw new IllegalArgumentException("password may not be null");
+ }
+
+ BufferedInputStream bis = null;
+ final KeyStore ks;
+ try {
+
+ // load the keystore
+ bis = new BufferedInputStream(keystore.openStream());
+ ks = KeyStore.getInstance(keystoreType.name());
+ ks.load(bis, password);
+
+ return true;
+
+ } catch (Exception e) {
+ return false;
+ } finally {
+ if (bis != null) {
+ try {
+ bis.close();
+ } catch (final IOException ioe) {
+ logger.warn("Failed to close input stream", ioe);
+ }
+ }
+ }
+ }
+
+ /**
+ * Extracts the username from the specified DN. If the username cannot be
+ * extracted because the CN is in an unrecognized format, the entire CN is
+ * returned. If the CN cannot be extracted because the DN is in an
+ * unrecognized format, the entire DN is returned.
+ *
+ * @param dn the dn to extract the username from
+ * @return the exatracted username
+ */
+ public static String extractUsername(String dn) {
+ String username = dn;
+ String cn = "";
+
+ // ensure the dn is specified
+ if (StringUtils.isNotBlank(dn)) {
+
+ // attempt to locate the cn
+ if (dn.startsWith("CN=")) {
+ cn = StringUtils.substringBetween(dn, "CN=", ",");
+ } else if (dn.startsWith("/CN=")) {
+ cn = StringUtils.substringBetween(dn, "CN=", "/");
+ } else if (dn.startsWith("C=") || dn.startsWith("/C=")) {
+ cn = StringUtils.substringAfter(dn, "CN=");
+ } else if (dn.startsWith("/") && StringUtils.contains(dn, "CN=")) {
+ cn = StringUtils.substringAfter(dn, "CN=");
+ }
+
+ // attempt to get the username from the cn
+ if (StringUtils.isNotBlank(cn)) {
+ if (cn.endsWith(")")) {
+ username = StringUtils.substringBetween(cn, "(", ")");
+ } else if (cn.contains(" ")) {
+ username = StringUtils.substringAfterLast(cn, " ");
+ } else {
+ username = cn;
+ }
+ }
+ }
+
+ return username;
+ }
+
+ /**
+ * Returns a list of subject alternative names. Any name that is represented
+ * as a String by X509Certificate.getSubjectAlternativeNames() is converted
+ * to lowercase and returned.
+ *
+ * @param certificate a certificate
+ * @return a list of subject alternative names; list is never null
+ * @throws CertificateParsingException if parsing the certificate failed
+ */
+ public static List<String> getSubjectAlternativeNames(final X509Certificate certificate) throws CertificateParsingException {
+
+ final Collection<List<?>> altNames = certificate.getSubjectAlternativeNames();
+ if (altNames == null) {
+ return new ArrayList<>();
+ }
+
+ final List<String> result = new ArrayList<>();
+ for (final List<?> generalName : altNames) {
+ /**
+ * generalName has the name type as the first element a String or
+ * byte array for the second element. We return any general names
+ * that are String types.
+ *
+ * We don't inspect the numeric name type because some certificates
+ * incorrectly put IPs and DNS names under the wrong name types.
+ */
+ final Object value = generalName.get(1);
+ if (value instanceof String) {
+ result.add(((String) value).toLowerCase());
+ }
+
+ }
+
+ return result;
+ }
+
+ private CertificateUtils() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/EncryptionMethod.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/EncryptionMethod.java b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/EncryptionMethod.java
new file mode 100644
index 0000000..55f5986
--- /dev/null
+++ b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/EncryptionMethod.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.security.util;
+
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
+
+/**
+ * Enumeration capturing essential information about the various encryption
+ * methods that might be supported.
+ *
+ */
+public enum EncryptionMethod {
+
+ MD5_128AES("PBEWITHMD5AND128BITAES-CBC-OPENSSL", "BC", false),
+ MD5_256AES("PBEWITHMD5AND256BITAES-CBC-OPENSSL", "BC", false),
+ SHA1_RC2("PBEWITHSHA1ANDRC2", "BC", false),
+ SHA1_DES("PBEWITHSHA1ANDDES", "BC", false),
+ MD5_192AES("PBEWITHMD5AND192BITAES-CBC-OPENSSL", "BC", false),
+ MD5_DES("PBEWITHMD5ANDDES", "BC", false),
+ MD5_RC2("PBEWITHMD5ANDRC2", "BC", false),
+ SHA_192AES("PBEWITHSHAAND192BITAES-CBC-BC", "BC", true),
+ SHA_40RC4("PBEWITHSHAAND40BITRC4", "BC", true),
+ SHA256_128AES("PBEWITHSHA256AND128BITAES-CBC-BC", "BC", true),
+ SHA_128RC2("PBEWITHSHAAND128BITRC2-CBC", "BC", true),
+ SHA_128AES("PBEWITHSHAAND128BITAES-CBC-BC", "BC", true),
+ SHA256_192AES("PBEWITHSHA256AND192BITAES-CBC-BC", "BC", true),
+ SHA_2KEYTRIPLEDES("PBEWITHSHAAND2-KEYTRIPLEDES-CBC", "BC", true),
+ SHA256_256AES("PBEWITHSHA256AND256BITAES-CBC-BC", "BC", true),
+ SHA_40RC2("PBEWITHSHAAND40BITRC2-CBC", "BC", true),
+ SHA_256AES("PBEWITHSHAAND256BITAES-CBC-BC", "BC", true),
+ SHA_3KEYTRIPLEDES("PBEWITHSHAAND3-KEYTRIPLEDES-CBC", "BC", true),
+ SHA_TWOFISH("PBEWITHSHAANDTWOFISH-CBC", "BC", true),
+ SHA_128RC4("PBEWITHSHAAND128BITRC4", "BC", true),
+ PGP("PGP", "BC", false),
+ PGP_ASCII_ARMOR("PGP-ASCII-ARMOR", "BC", false);
+
+ private final String algorithm;
+ private final String provider;
+ private final boolean unlimitedStrength;
+
+ EncryptionMethod(String algorithm, String provider, boolean unlimitedStrength) {
+ this.algorithm = algorithm;
+ this.provider = provider;
+ this.unlimitedStrength = unlimitedStrength;
+ }
+
+ public String getProvider() {
+ return provider;
+ }
+
+ public String getAlgorithm() {
+ return algorithm;
+ }
+
+ /**
+ * @return true if algorithm requires unlimited strength policies
+ */
+ public boolean isUnlimitedStrength() {
+ return unlimitedStrength;
+ }
+
+ @Override
+ public String toString() {
+ final ToStringBuilder builder = new ToStringBuilder(this);
+ ToStringBuilder.setDefaultStyle(ToStringStyle.SHORT_PREFIX_STYLE);
+ builder.append("algorithm name", algorithm);
+ builder.append("Requires unlimited strength JCE policy", unlimitedStrength);
+ builder.append("Algorithm Provider", provider);
+ return builder.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/KeystoreType.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/KeystoreType.java b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/KeystoreType.java
new file mode 100644
index 0000000..18574bb
--- /dev/null
+++ b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/KeystoreType.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.security.util;
+
+/**
+ * Keystore types.
+ */
+public enum KeystoreType {
+
+ PKCS12,
+ JKS;
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/SecurityStoreTypes.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/SecurityStoreTypes.java b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/SecurityStoreTypes.java
new file mode 100644
index 0000000..f448312
--- /dev/null
+++ b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/SecurityStoreTypes.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.security.util;
+
+import java.io.PrintWriter;
+import java.io.Writer;
+
+/**
+ * Types of security stores and their related Java system properties.
+ */
+public enum SecurityStoreTypes {
+
+ TRUSTSTORE(
+ "javax.net.ssl.trustStore",
+ "javax.net.ssl.trustStorePassword",
+ "javax.net.ssl.trustStoreType"),
+ KEYSTORE(
+ "javax.net.ssl.keyStore",
+ "javax.net.ssl.keyStorePassword",
+ "javax.net.ssl.keyStoreType");
+
+ /**
+ * Logs the keystore and truststore Java system property values to the given
+ * writer. It logPasswords is true, then the keystore and truststore
+ * password property values are logged.
+ *
+ * @param writer a writer to log to
+ *
+ * @param logPasswords true if passwords should be logged; false otherwise
+ */
+ public static void logProperties(final Writer writer,
+ final boolean logPasswords) {
+ if (writer == null) {
+ return;
+ }
+
+ PrintWriter pw = new PrintWriter(writer);
+
+ // keystore properties
+ pw.println(
+ KEYSTORE.getStoreProperty() + " = " + System.getProperty(KEYSTORE.getStoreProperty()));
+
+ if (logPasswords) {
+ pw.println(
+ KEYSTORE.getStorePasswordProperty() + " = "
+ + System.getProperty(KEYSTORE.getStoreProperty()));
+ }
+
+ pw.println(
+ KEYSTORE.getStoreTypeProperty() + " = "
+ + System.getProperty(KEYSTORE.getStoreTypeProperty()));
+
+ // truststore properties
+ pw.println(
+ TRUSTSTORE.getStoreProperty() + " = "
+ + System.getProperty(TRUSTSTORE.getStoreProperty()));
+
+ if (logPasswords) {
+ pw.println(
+ TRUSTSTORE.getStorePasswordProperty() + " = "
+ + System.getProperty(TRUSTSTORE.getStoreProperty()));
+ }
+
+ pw.println(
+ TRUSTSTORE.getStoreTypeProperty() + " = "
+ + System.getProperty(TRUSTSTORE.getStoreTypeProperty()));
+ pw.flush();
+ }
+
+ /**
+ * the Java system property for setting the keystore (or truststore) path
+ */
+ private String storeProperty = "";
+
+ /**
+ * the Java system property for setting the keystore (or truststore)
+ * password
+ */
+ private String storePasswordProperty = "";
+
+ /**
+ * the Java system property for setting the keystore (or truststore) type
+ */
+ private String storeTypeProperty = "";
+
+ /**
+ * Creates an instance.
+ *
+ * @param storeProperty the Java system property for setting the keystore or
+ * truststore path
+ * @param storePasswordProperty the Java system property for setting the
+ * keystore or truststore path
+ * @param storeTypeProperty the Java system property for setting the
+ * keystore or truststore type
+ */
+ SecurityStoreTypes(final String storeProperty,
+ final String storePasswordProperty,
+ final String storeTypeProperty) {
+ this.storeProperty = storeProperty;
+ this.storePasswordProperty = storePasswordProperty;
+ this.storeTypeProperty = storeTypeProperty;
+ }
+
+ /**
+ * Returns the keystore (or truststore) property.
+ *
+ * @return the keystore (or truststore) property
+ */
+ public String getStoreProperty() {
+ return storeProperty;
+ }
+
+ /**
+ * Returns the keystore (or truststore) password property.
+ *
+ * @return the keystore (or truststore) password property
+ */
+ public String getStorePasswordProperty() {
+ return storePasswordProperty;
+ }
+
+ /**
+ * Returns the keystore (or truststore) type property.
+ *
+ * @return the keystore (or truststore) type property
+ */
+ public String getStoreTypeProperty() {
+ return storeTypeProperty;
+ }
+}
[35/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ToUpperEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ToUpperEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ToUpperEvaluator.java
new file mode 100644
index 0000000..f3174c2
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ToUpperEvaluator.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult;
+
+public class ToUpperEvaluator extends StringEvaluator {
+
+ private final Evaluator<String> subject;
+
+ public ToUpperEvaluator(final Evaluator<String> subject) {
+ this.subject = subject;
+ }
+
+ @Override
+ public QueryResult<String> evaluate(final Map<String, String> attributes) {
+ final String subjectValue = subject.evaluate(attributes).getValue();
+ return new StringQueryResult(subjectValue == null ? null : subjectValue.toUpperCase());
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/TrimEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/TrimEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/TrimEvaluator.java
new file mode 100644
index 0000000..e44661f
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/TrimEvaluator.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult;
+
+public class TrimEvaluator extends StringEvaluator {
+
+ private final Evaluator<String> subject;
+
+ public TrimEvaluator(final Evaluator<String> subject) {
+ this.subject = subject;
+ }
+
+ @Override
+ public QueryResult<String> evaluate(Map<String, String> attributes) {
+ final String subjectValue = subject.evaluate(attributes).getValue();
+ return new StringQueryResult(subjectValue == null ? null : subjectValue.trim());
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/UrlDecodeEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/UrlDecodeEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/UrlDecodeEvaluator.java
new file mode 100644
index 0000000..c61d55d
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/UrlDecodeEvaluator.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult;
+
+public class UrlDecodeEvaluator extends StringEvaluator {
+
+ private final Evaluator<String> subject;
+
+ public UrlDecodeEvaluator(final Evaluator<String> subject) {
+ this.subject = subject;
+ }
+
+ @Override
+ public QueryResult<String> evaluate(final Map<String, String> attributes) {
+ final String subjectValue = subject.evaluate(attributes).getValue();
+ if (subjectValue == null) {
+ return new StringQueryResult(null);
+ }
+
+ try {
+ return new StringQueryResult(URLDecoder.decode(subjectValue, "UTF-8"));
+ } catch (final UnsupportedEncodingException e) {
+ return null; // won't happen. It's UTF-8
+ }
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/UrlEncodeEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/UrlEncodeEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/UrlEncodeEvaluator.java
new file mode 100644
index 0000000..769f6df
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/UrlEncodeEvaluator.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult;
+
+public class UrlEncodeEvaluator extends StringEvaluator {
+
+ private final Evaluator<String> subject;
+
+ public UrlEncodeEvaluator(final Evaluator<String> subject) {
+ this.subject = subject;
+ }
+
+ @Override
+ public QueryResult<String> evaluate(final Map<String, String> attributes) {
+ final String subjectValue = subject.evaluate(attributes).getValue();
+ if (subjectValue == null) {
+ return new StringQueryResult(null);
+ }
+
+ try {
+ return new StringQueryResult(URLEncoder.encode(subjectValue, "UTF-8"));
+ } catch (final UnsupportedEncodingException e) {
+ return null; // won't happen. It's UTF-8
+ }
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/UuidEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/UuidEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/UuidEvaluator.java
new file mode 100644
index 0000000..faa8f0a
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/UuidEvaluator.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+import java.util.UUID;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult;
+
+public class UuidEvaluator extends StringEvaluator {
+
+ @Override
+ public QueryResult<String> evaluate(final Map<String, String> attributes) {
+ return new StringQueryResult(UUID.randomUUID().toString());
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/literals/BooleanLiteralEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/literals/BooleanLiteralEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/literals/BooleanLiteralEvaluator.java
new file mode 100644
index 0000000..a085423
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/literals/BooleanLiteralEvaluator.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.literals;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+
+public class BooleanLiteralEvaluator extends BooleanEvaluator {
+
+ private final boolean value;
+
+ public BooleanLiteralEvaluator(final boolean value) {
+ this.value = value;
+ }
+
+ @Override
+ public QueryResult<Boolean> evaluate(final Map<String, String> attributes) {
+ return new BooleanQueryResult(value);
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/literals/NumberLiteralEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/literals/NumberLiteralEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/literals/NumberLiteralEvaluator.java
new file mode 100644
index 0000000..d7569e0
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/literals/NumberLiteralEvaluator.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.literals;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.NumberEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.NumberQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+
+public class NumberLiteralEvaluator extends NumberEvaluator {
+
+ private final long literal;
+
+ public NumberLiteralEvaluator(final String value) {
+ this.literal = Long.parseLong(value);
+ }
+
+ @Override
+ public QueryResult<Long> evaluate(final Map<String, String> attributes) {
+ return new NumberQueryResult(literal);
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/literals/StringLiteralEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/literals/StringLiteralEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/literals/StringLiteralEvaluator.java
new file mode 100644
index 0000000..d739ac7
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/literals/StringLiteralEvaluator.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.literals;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult;
+
+public class StringLiteralEvaluator extends StringEvaluator {
+
+ private final String value;
+
+ public StringLiteralEvaluator(final String value) {
+ // need to escape characters after backslashes
+ final StringBuilder sb = new StringBuilder();
+ boolean lastCharIsBackslash = false;
+ for (int i = 0; i < value.length(); i++) {
+ final char c = value.charAt(i);
+
+ if (lastCharIsBackslash) {
+ switch (c) {
+ case 'n':
+ sb.append("\n");
+ break;
+ case 'r':
+ sb.append("\r");
+ break;
+ case '\\':
+ sb.append("\\");
+ break;
+ case 't':
+ sb.append("\\t");
+ break;
+ default:
+ sb.append("\\").append(c);
+ break;
+ }
+
+ lastCharIsBackslash = false;
+ } else if (c == '\\') {
+ lastCharIsBackslash = true;
+ } else {
+ sb.append(c);
+ }
+ }
+
+ this.value = sb.toString();
+ }
+
+ @Override
+ public QueryResult<String> evaluate(final Map<String, String> attributes) {
+ return new StringQueryResult(value);
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/literals/ToLiteralEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/literals/ToLiteralEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/literals/ToLiteralEvaluator.java
new file mode 100644
index 0000000..e297eea
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/literals/ToLiteralEvaluator.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.literals;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult;
+
+public class ToLiteralEvaluator extends StringEvaluator {
+ private final Evaluator<?> argEvaluator;
+
+ public ToLiteralEvaluator(final Evaluator<?> argEvaluator) {
+ this.argEvaluator = argEvaluator;
+ }
+
+ @Override
+ public QueryResult<String> evaluate(final Map<String, String> attributes) {
+ final Object result = argEvaluator.evaluate(attributes);
+ return new StringQueryResult(result == null ? null : result.toString());
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/reduce/CountEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/reduce/CountEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/reduce/CountEvaluator.java
new file mode 100644
index 0000000..f3fb21d
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/reduce/CountEvaluator.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.reduce;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.NumberEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.NumberQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.expression.AttributeExpression.ResultType;
+
+public class CountEvaluator extends NumberEvaluator implements ReduceEvaluator<Long> {
+
+ private final Evaluator<?> subjectEvaluator;
+ private long count = 0L;
+
+ public CountEvaluator(final Evaluator<?> subjectEvaluator) {
+ this.subjectEvaluator = subjectEvaluator;
+ }
+
+ @Override
+ public QueryResult<Long> evaluate(final Map<String, String> attributes) {
+ final QueryResult<?> result = subjectEvaluator.evaluate(attributes);
+ if (result.getValue() == null) {
+ return new NumberQueryResult(count);
+ }
+
+ if (result.getResultType() == ResultType.BOOLEAN && ((Boolean) result.getValue()).equals(Boolean.FALSE)) {
+ return new NumberQueryResult(count);
+ }
+
+ count++;
+ return new NumberQueryResult(count);
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subjectEvaluator;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/reduce/JoinEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/reduce/JoinEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/reduce/JoinEvaluator.java
new file mode 100644
index 0000000..f80924b
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/reduce/JoinEvaluator.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.reduce;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult;
+
+public class JoinEvaluator extends StringEvaluator implements ReduceEvaluator<String> {
+
+ private final Evaluator<String> subjectEvaluator;
+ private final Evaluator<String> delimiterEvaluator;
+
+ private final StringBuilder sb = new StringBuilder();
+ private int evalCount = 0;
+
+ public JoinEvaluator(final Evaluator<String> subject, final Evaluator<String> delimiter) {
+ this.subjectEvaluator = subject;
+ this.delimiterEvaluator = delimiter;
+ }
+
+ @Override
+ public QueryResult<String> evaluate(final Map<String, String> attributes) {
+ String subject = subjectEvaluator.evaluate(attributes).getValue();
+ if (subject == null) {
+ subject = "";
+ }
+
+ final String delimiter = delimiterEvaluator.evaluate(attributes).getValue();
+ if (evalCount > 0) {
+ sb.append(delimiter);
+ }
+ sb.append(subject);
+
+ evalCount++;
+ return new StringQueryResult(sb.toString());
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subjectEvaluator;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/reduce/ReduceEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/reduce/ReduceEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/reduce/ReduceEvaluator.java
new file mode 100644
index 0000000..12197c0
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/reduce/ReduceEvaluator.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.reduce;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+
+public interface ReduceEvaluator<T> extends Evaluator<T> {
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/AllAttributesEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/AllAttributesEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/AllAttributesEvaluator.java
new file mode 100644
index 0000000..ccf9d19
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/AllAttributesEvaluator.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.selection;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+
+public class AllAttributesEvaluator extends BooleanEvaluator implements IteratingEvaluator<Boolean> {
+
+ private final BooleanEvaluator booleanEvaluator;
+ private final MultiAttributeEvaluator multiAttributeEvaluator;
+
+ public AllAttributesEvaluator(final BooleanEvaluator booleanEvaluator, final MultiAttributeEvaluator multiAttributeEvaluator) {
+ this.booleanEvaluator = booleanEvaluator;
+ this.multiAttributeEvaluator = multiAttributeEvaluator;
+ }
+
+ @Override
+ public QueryResult<Boolean> evaluate(final Map<String, String> attributes) {
+ QueryResult<Boolean> attributeValueQuery = booleanEvaluator.evaluate(attributes);
+ Boolean result = attributeValueQuery.getValue();
+ if (result == null) {
+ return new BooleanQueryResult(false);
+ }
+
+ if (!result) {
+ return new BooleanQueryResult(false);
+ }
+
+ while (multiAttributeEvaluator.getEvaluationsRemaining() > 0) {
+ attributeValueQuery = booleanEvaluator.evaluate(attributes);
+ result = attributeValueQuery.getValue();
+ if (result != null && !result) {
+ return attributeValueQuery;
+ }
+ }
+
+ return new BooleanQueryResult(true);
+ }
+
+ @Override
+ public int getEvaluationsRemaining() {
+ return 0;
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return null;
+ }
+
+ @Override
+ public Evaluator<?> getLogicEvaluator() {
+ return booleanEvaluator;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/AnyAttributeEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/AnyAttributeEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/AnyAttributeEvaluator.java
new file mode 100644
index 0000000..6e8d485
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/AnyAttributeEvaluator.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.selection;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+
+public class AnyAttributeEvaluator extends BooleanEvaluator implements IteratingEvaluator<Boolean> {
+
+ private final BooleanEvaluator booleanEvaluator;
+ private final MultiAttributeEvaluator multiAttributeEvaluator;
+
+ public AnyAttributeEvaluator(final BooleanEvaluator booleanEvaluator, final MultiAttributeEvaluator multiAttributeEvaluator) {
+ this.booleanEvaluator = booleanEvaluator;
+ this.multiAttributeEvaluator = multiAttributeEvaluator;
+ }
+
+ @Override
+ public QueryResult<Boolean> evaluate(final Map<String, String> attributes) {
+ QueryResult<Boolean> attributeValueQuery = booleanEvaluator.evaluate(attributes);
+ Boolean result = attributeValueQuery.getValue();
+ if (result == null) {
+ return new BooleanQueryResult(false);
+ }
+
+ if (result) {
+ return new BooleanQueryResult(true);
+ }
+
+ while (multiAttributeEvaluator.getEvaluationsRemaining() > 0) {
+ attributeValueQuery = booleanEvaluator.evaluate(attributes);
+ result = attributeValueQuery.getValue();
+ if (result != null && result) {
+ return attributeValueQuery;
+ }
+ }
+
+ return new BooleanQueryResult(false);
+ }
+
+ @Override
+ public int getEvaluationsRemaining() {
+ return 0;
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return null;
+ }
+
+ @Override
+ public Evaluator<Boolean> getLogicEvaluator() {
+ return booleanEvaluator;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/DelineatedAttributeEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/DelineatedAttributeEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/DelineatedAttributeEvaluator.java
new file mode 100644
index 0000000..e39d01c
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/DelineatedAttributeEvaluator.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.selection;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult;
+
+public class DelineatedAttributeEvaluator extends MultiAttributeEvaluator {
+
+ private final Evaluator<String> subjectEvaluator;
+ private final Evaluator<String> delimiterEvaluator;
+ private final int evaluationType;
+ private String[] delineatedValues;
+ private int evaluationCount = 0;
+ private int evaluationsLeft = 1;
+
+ public DelineatedAttributeEvaluator(final Evaluator<String> subjectEvaluator, final Evaluator<String> delimiterEvaluator, final int evaluationType) {
+ this.subjectEvaluator = subjectEvaluator;
+ this.delimiterEvaluator = delimiterEvaluator;
+ this.evaluationType = evaluationType;
+ }
+
+ @Override
+ public QueryResult<String> evaluate(final Map<String, String> attributes) {
+ if (delineatedValues == null) {
+ final QueryResult<String> subjectValue = subjectEvaluator.evaluate(attributes);
+ if (subjectValue.getValue() == null) {
+ evaluationsLeft = 0;
+ return new StringQueryResult(null);
+ }
+
+ final QueryResult<String> delimiterValue = delimiterEvaluator.evaluate(attributes);
+ if (subjectValue.getValue() == null) {
+ evaluationsLeft = 0;
+ return new StringQueryResult(null);
+ }
+
+ delineatedValues = subjectValue.getValue().split(delimiterValue.getValue());
+ }
+
+ if (evaluationCount > delineatedValues.length) {
+ evaluationsLeft = 0;
+ return new StringQueryResult(null);
+ }
+
+ evaluationsLeft = delineatedValues.length - evaluationCount - 1;
+
+ return new StringQueryResult(delineatedValues[evaluationCount++]);
+ }
+
+ @Override
+ public Evaluator<?> getLogicEvaluator() {
+ return subjectEvaluator;
+ }
+
+ @Override
+ public int getEvaluationsRemaining() {
+ return evaluationsLeft;
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return null;
+ }
+
+ @Override
+ public int getEvaluationType() {
+ return evaluationType;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/IteratingEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/IteratingEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/IteratingEvaluator.java
new file mode 100644
index 0000000..5156345
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/IteratingEvaluator.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.selection;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+
+/**
+ * Interface for an Evaluator that should be evaluated multiple times
+ *
+ * @param <T> return type of evaluator
+ */
+public interface IteratingEvaluator<T> extends Evaluator<T> {
+
+ /**
+ * @return the evaluator that evaluates some sort of logic against its subject
+ */
+ Evaluator<?> getLogicEvaluator();
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/MappingEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/MappingEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/MappingEvaluator.java
new file mode 100644
index 0000000..e007a56
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/MappingEvaluator.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.selection;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.reduce.ReduceEvaluator;
+import org.apache.nifi.expression.AttributeExpression.ResultType;
+
+public class MappingEvaluator<T> implements Evaluator<T> {
+ private final ReduceEvaluator<T> mappingEvaluator;
+ private final MultiAttributeEvaluator multiAttributeEvaluator;
+ private String token;
+
+ public MappingEvaluator(final ReduceEvaluator<T> mappingEvaluator, final MultiAttributeEvaluator multiAttributeEval) {
+ this.mappingEvaluator = mappingEvaluator;
+ this.multiAttributeEvaluator = multiAttributeEval;
+ }
+
+ @Override
+ public QueryResult<T> evaluate(final Map<String, String> attributes) {
+ QueryResult<T> result = mappingEvaluator.evaluate(attributes);
+
+ while (multiAttributeEvaluator.getEvaluationsRemaining() > 0) {
+ result = mappingEvaluator.evaluate(attributes);
+ }
+
+ return result;
+ }
+
+ @Override
+ public ResultType getResultType() {
+ return mappingEvaluator.getResultType();
+ }
+
+ @Override
+ public int getEvaluationsRemaining() {
+ return 0;
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return null;
+ }
+
+ @Override
+ public String getToken() {
+ return token;
+ }
+
+ @Override
+ public void setToken(final String token) {
+ this.token = token;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/MultiAttributeEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/MultiAttributeEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/MultiAttributeEvaluator.java
new file mode 100644
index 0000000..78b5af4
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/MultiAttributeEvaluator.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.selection;
+
+import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator;
+
+public abstract class MultiAttributeEvaluator extends StringEvaluator implements IteratingEvaluator<String> {
+ public abstract int getEvaluationType();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/MultiMatchAttributeEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/MultiMatchAttributeEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/MultiMatchAttributeEvaluator.java
new file mode 100644
index 0000000..8292f84
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/MultiMatchAttributeEvaluator.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.selection;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult;
+
+public class MultiMatchAttributeEvaluator extends MultiAttributeEvaluator {
+
+ private final List<Pattern> attributePatterns;
+ private final int evaluationType;
+ private final List<String> attributeNames = new ArrayList<>();
+ private int evaluationCount = 0;
+
+ public MultiMatchAttributeEvaluator(final List<String> attributeRegexes, final int evaluationType) {
+ this.attributePatterns = new ArrayList<>();
+ for (final String regex : attributeRegexes) {
+ attributePatterns.add(Pattern.compile(regex));
+ }
+
+ this.evaluationType = evaluationType;
+ }
+
+ /**
+ * Can be called only after the first call to evaluate
+ *
+ * @return number of remaining evaluations
+ */
+ @Override
+ public int getEvaluationsRemaining() {
+ return attributeNames.size() - evaluationCount;
+ }
+
+ @Override
+ public QueryResult<String> evaluate(final Map<String, String> attributes) {
+ if (evaluationCount == 0) {
+ for (final Pattern pattern : attributePatterns) {
+ for (final String attrName : attributes.keySet()) {
+ if (pattern.matcher(attrName).matches()) {
+ attributeNames.add(attrName);
+ }
+ }
+ }
+ }
+
+ if (evaluationCount >= attributeNames.size()) {
+ return new StringQueryResult(null);
+ }
+
+ return new StringQueryResult(attributes.get(attributeNames.get(evaluationCount++)));
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return null;
+ }
+
+ @Override
+ public int getEvaluationType() {
+ return evaluationType;
+ }
+
+ @Override
+ public Evaluator<?> getLogicEvaluator() {
+ return this;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/MultiNamedAttributeEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/MultiNamedAttributeEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/MultiNamedAttributeEvaluator.java
new file mode 100644
index 0000000..509d7dd
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/MultiNamedAttributeEvaluator.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.selection;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult;
+
+public class MultiNamedAttributeEvaluator extends MultiAttributeEvaluator {
+
+ private final List<String> attributeNames;
+ private final int evaluationType;
+ private int evaluationCount = 0;
+ private List<String> matchingAttributeNames = null;
+
+ public MultiNamedAttributeEvaluator(final List<String> attributeNames, final int evaluationType) {
+ this.attributeNames = attributeNames;
+ this.evaluationType = evaluationType;
+ }
+
+ @Override
+ public QueryResult<String> evaluate(final Map<String, String> attributes) {
+ matchingAttributeNames = new ArrayList<>(attributeNames);
+
+ if (matchingAttributeNames.size() <= evaluationCount) {
+ return new StringQueryResult(null);
+ }
+
+ return new StringQueryResult(attributes.get(matchingAttributeNames.get(evaluationCount++)));
+ }
+
+ @Override
+ public int getEvaluationsRemaining() {
+ return matchingAttributeNames.size() - evaluationCount;
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return null;
+ }
+
+ @Override
+ public int getEvaluationType() {
+ return evaluationType;
+ }
+
+ @Override
+ public Evaluator<?> getLogicEvaluator() {
+ return this;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/exception/AttributeExpressionLanguageException.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/exception/AttributeExpressionLanguageException.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/exception/AttributeExpressionLanguageException.java
new file mode 100644
index 0000000..47d42cb
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/exception/AttributeExpressionLanguageException.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.exception;
+
+public class AttributeExpressionLanguageException extends RuntimeException {
+
+ private static final long serialVersionUID = -5637284498692447901L;
+
+ public AttributeExpressionLanguageException(final String explanation) {
+ super(explanation);
+ }
+
+ public AttributeExpressionLanguageException(final String explanation, final Throwable t) {
+ super(explanation, t);
+ }
+
+ public AttributeExpressionLanguageException(final Throwable t) {
+ super(t);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/exception/AttributeExpressionLanguageParsingException.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/exception/AttributeExpressionLanguageParsingException.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/exception/AttributeExpressionLanguageParsingException.java
new file mode 100644
index 0000000..f8531cb
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/exception/AttributeExpressionLanguageParsingException.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.exception;
+
+public class AttributeExpressionLanguageParsingException extends AttributeExpressionLanguageException {
+
+ private static final long serialVersionUID = 7422163230677064726L;
+
+ public AttributeExpressionLanguageParsingException(final String explanation) {
+ super(explanation);
+ }
+
+ public AttributeExpressionLanguageParsingException(final String explanation, final Throwable t) {
+ super(explanation, t);
+ }
+
+ public AttributeExpressionLanguageParsingException(final Throwable t) {
+ super(t);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/exception/IllegalAttributeException.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/exception/IllegalAttributeException.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/exception/IllegalAttributeException.java
new file mode 100644
index 0000000..ec94837
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/exception/IllegalAttributeException.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.exception;
+
+public class IllegalAttributeException extends RuntimeException {
+
+ private static final long serialVersionUID = 12348721897342L;
+
+ public IllegalAttributeException() {
+ super();
+ }
+
+ public IllegalAttributeException(final String explanation) {
+ super(explanation);
+ }
+}
[48/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/components/PropertyValue.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/components/PropertyValue.java b/nifi-api/src/main/java/org/apache/nifi/components/PropertyValue.java
new file mode 100644
index 0000000..1845ed2
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/components/PropertyValue.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.components;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.nifi.controller.ControllerService;
+import org.apache.nifi.expression.AttributeValueDecorator;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.DataUnit;
+import org.apache.nifi.processor.exception.ProcessException;
+
+/**
+ * <p>
+ * A PropertyValue provides a mechanism whereby the currently configured value
+ * of a processor property can be obtained in different forms.
+ * </p>
+ */
+public interface PropertyValue {
+
+ /**
+ * @return the raw property value as a string
+ */
+ public String getValue();
+
+ /**
+ * @return an integer representation of the property value, or
+ * <code>null</code> if not set
+ * @throws NumberFormatException if not able to parse
+ */
+ public Integer asInteger();
+
+ /**
+ * @return a Long representation of the property value, or <code>null</code>
+ * if not set
+ * @throws NumberFormatException if not able to parse
+ */
+ public Long asLong();
+
+ /**
+ * @return a Boolean representation of the property value, or
+ * <code>null</code> if not set
+ */
+ public Boolean asBoolean();
+
+ /**
+ * @return a Float representation of the property value, or
+ * <code>null</code> if not set
+ * @throws NumberFormatException if not able to parse
+ */
+ public Float asFloat();
+
+ /**
+ * @return a Double representation of the property value, of
+ * <code>null</code> if not set
+ * @throws NumberFormatException if not able to parse
+ */
+ public Double asDouble();
+
+ /**
+ * @param timeUnit specifies the TimeUnit to convert the time duration into
+ * @return a Long value representing the value of the configured time period
+ * in terms of the specified TimeUnit; if the property is not set, returns
+ * <code>null</code>
+ */
+ public Long asTimePeriod(TimeUnit timeUnit);
+
+ /**
+ *
+ * @param dataUnit specifies the DataUnit to convert the data size into
+ * @return a Long value representing the value of the configured data size
+ * in terms of the specified DataUnit; if hte property is not set, returns
+ * <code>null</code>
+ */
+ public Double asDataSize(DataUnit dataUnit);
+
+ /**
+ * @return the ControllerService whose identifier is the raw value of
+ * <code>this</code>, or <code>null</code> if either the value is not set or
+ * the value does not identify a ControllerService
+ */
+ public ControllerService asControllerService();
+
+ /**
+ * @param <T> the generic type of the controller service
+ * @param serviceType the class of the Controller Service
+ * @return the ControllerService whose identifier is the raw value of the
+ * <code>this</code>, or <code>null</code> if either the value is not set or
+ * the value does not identify a ControllerService. The object returned by
+ * this method is explicitly cast to type specified, if the type specified
+ * is valid. Otherwise, throws an IllegalArgumentException
+ *
+ * @throws IllegalArgumentException if the value of <code>this</code> points
+ * to a ControllerService but that service is not of type
+ * <code>serviceType</code> or if <code>serviceType</code> references a
+ * class that is not an interface
+ */
+ public <T extends ControllerService> T asControllerService(Class<T> serviceType) throws IllegalArgumentException;
+
+ /**
+ * @return <code>true</code> if the user has configured a value, or if the
+ * {@link PropertyDescriptor} for the associated property has a default
+ * value, <code>false</code> otherwise
+ */
+ public boolean isSet();
+
+ /**
+ * <p>
+ * Replaces values in the Property Value using the Attribute Expression
+ * Language; a PropertyValue with the new value is then returned, supporting
+ * call chaining.
+ * </p>
+ *
+ * @return a PropertyValue with the new value is returned, supporting call
+ * chaining
+ *
+ * @throws ProcessException if the Query cannot be compiled or evaluating
+ * the query against the given attributes causes an Exception to be thrown
+ */
+ public PropertyValue evaluateAttributeExpressions() throws ProcessException;
+
+ /**
+ * <p>
+ * Replaces values in the Property Value using the Attribute Expression
+ * Language; a PropertyValue with the new value is then returned, supporting
+ * call chaining.
+ * </p>
+ *
+ * @param flowFile to evaluate attributes of
+ * @return a PropertyValue with the new value is returned, supporting call
+ * chaining
+ *
+ * @throws ProcessException if the Query cannot be compiled or evaluating
+ * the query against the given attributes causes an Exception to be thrown
+ */
+ public PropertyValue evaluateAttributeExpressions(FlowFile flowFile) throws ProcessException;
+
+ /**
+ * <p>
+ * Replaces values in the Property Value using the Attribute Expression
+ * Language. The supplied decorator is then given a chance to decorate the
+ * value, and a PropertyValue with the new value is then returned,
+ * supporting call chaining.
+ * </p>
+ *
+ * @param decorator The supplied decorator is then given a chance to
+ * decorate the value
+ * @return a PropertyValue with the new value is then returned, supporting
+ * call chaining
+ *
+ * @throws ProcessException if the Query cannot be compiled or evaluating
+ * the query against the given attributes causes an Exception to be thrown
+ */
+ public PropertyValue evaluateAttributeExpressions(AttributeValueDecorator decorator) throws ProcessException;
+
+ /**
+ * <p>
+ * Replaces values in the Property Value using the Attribute Expression
+ * Language. The supplied decorator is then given a chance to decorate the
+ * value, and a PropertyValue with the new value is then returned,
+ * supporting call chaining.
+ * </p>
+ *
+ * @param flowFile to evaluate expressions against
+ * @param decorator The supplied decorator is then given a chance to
+ * decorate the value
+ *
+ * @return a PropertyValue with the new value is then returned, supporting
+ * call chaining
+ *
+ * @throws ProcessException if the Query cannot be compiled or evaluating
+ * the query against the given attributes causes an Exception to be thrown
+ */
+ public PropertyValue evaluateAttributeExpressions(FlowFile flowFile, AttributeValueDecorator decorator) throws ProcessException;
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/components/ValidationContext.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/components/ValidationContext.java b/nifi-api/src/main/java/org/apache/nifi/components/ValidationContext.java
new file mode 100644
index 0000000..a1dcf43
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/components/ValidationContext.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.components;
+
+import java.util.Map;
+
+import org.apache.nifi.controller.ControllerService;
+import org.apache.nifi.controller.ControllerServiceLookup;
+import org.apache.nifi.expression.ExpressionLanguageCompiler;
+
+public interface ValidationContext {
+
+ /**
+ * @return the {@link ControllerServiceLookup} which can be used to obtain
+ * Controller Services
+ */
+ ControllerServiceLookup getControllerServiceLookup();
+
+ /**
+ * @param controllerService to lookup the validation context of
+ * @return a ValidationContext that is appropriate for validating the given
+ * {@link ControllerService}
+ */
+ ValidationContext getControllerServiceValidationContext(ControllerService controllerService);
+
+ /**
+ * @return a new {@link ExpressionLanguageCompiler} that can be used to
+ * compile & evaluate Attribute Expressions
+ */
+ ExpressionLanguageCompiler newExpressionLanguageCompiler();
+
+ /**
+ * @param property being validated
+ * @return a PropertyValue that encapsulates the value configured for the
+ * given PropertyDescriptor
+ */
+ PropertyValue getProperty(PropertyDescriptor property);
+
+ /**
+ * @param value to make a PropertyValue object for
+ * @return a PropertyValue that represents the given value
+ */
+ PropertyValue newPropertyValue(String value);
+
+ /**
+ * @return a Map of all configured Properties
+ */
+ Map<PropertyDescriptor, String> getProperties();
+
+ /**
+ * @return the currently configured Annotation Data
+ */
+ String getAnnotationData();
+
+ /**
+ * There are times when the framework needs to consider a component valid,
+ * even if it references an invalid ControllerService. This method will
+ * return <code>false</code> if the component is to be considered valid even
+ * if the given Controller Service is referenced and is invalid.
+ *
+ * @param service to check if validation is required
+ * @return <code>false</code> if the component is to be considered valid
+ * even if the given Controller Service is referenced and is invalid
+ */
+ boolean isValidationRequired(ControllerService service);
+
+ /**
+ * @param value to test whether expression language is present
+ * @return <code>true</code> if the given value contains a NiFi Expression
+ * Language expression, <code>false</code> if it does not
+ */
+ boolean isExpressionLanguagePresent(String value);
+
+ /**
+ * @param propertyName to test whether expression language is supported
+ * @return <code>true</code> if the property with the given name supports
+ * the NiFi Expression Language, <code>false</code> if the property does not
+ * support the Expression Language or is not a valid property name
+ */
+ boolean isExpressionLanguageSupported(String propertyName);
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/components/ValidationResult.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/components/ValidationResult.java b/nifi-api/src/main/java/org/apache/nifi/components/ValidationResult.java
new file mode 100644
index 0000000..e0beec8
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/components/ValidationResult.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.components;
+
+import java.util.Objects;
+
+/**
+ *
+ * Immutable - thread safe
+ *
+ */
+public class ValidationResult {
+
+ private final String subject;
+ private final String input;
+ private final String explanation;
+ private final boolean valid;
+
+ private ValidationResult(final Builder builder) {
+ this.subject = builder.subject;
+ this.input = builder.input;
+ this.explanation = builder.explanation;
+ this.valid = builder.valid;
+ }
+
+ /**
+ * @return true if current result is valid; false otherwise
+ */
+ public boolean isValid() {
+ return this.valid;
+ }
+
+ /**
+ * @return this input value that was tested for validity
+ */
+ public String getInput() {
+ return this.input;
+ }
+
+ /**
+ * @return an explanation of the validation result
+ */
+ public String getExplanation() {
+ return this.explanation;
+ }
+
+ /**
+ * @return the item being validated
+ */
+ public String getSubject() {
+ return this.subject;
+ }
+
+ @Override
+ public int hashCode() {
+ int hash = 3;
+ hash = 79 * hash + Objects.hashCode(this.subject);
+ hash = 79 * hash + Objects.hashCode(this.input);
+ hash = 79 * hash + Objects.hashCode(this.explanation);
+ hash = 79 * hash + (this.valid ? 1 : 0);
+ return hash;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ final ValidationResult other = (ValidationResult) obj;
+ if (!Objects.equals(this.subject, other.subject)) {
+ return false;
+ }
+ if (!Objects.equals(this.input, other.input)) {
+ return false;
+ }
+ if (!Objects.equals(this.explanation, other.explanation)) {
+ return false;
+ }
+ if (this.valid != other.valid) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public String toString() {
+ if (input == null) {
+ return String.format("'%s' is %s because %s", subject, (valid ? "valid" : "invalid"), explanation);
+ } else {
+ return String.format("'%s' validated against '%s' is %s because %s", subject, input, (valid ? "valid" : "invalid"), explanation);
+ }
+ }
+
+ public static final class Builder {
+
+ private boolean valid = false;
+ private String input = null;
+ private String explanation = "";
+ private String subject = "";
+
+ /**
+ * Defaults to false
+ *
+ * @param valid true if is valid; false otherwise
+ * @return the builder
+ */
+ public Builder valid(final boolean valid) {
+ this.valid = valid;
+ return this;
+ }
+
+ /**
+ * Defaults to empty string
+ *
+ * @param input what was validated
+ * @return the builder
+ */
+ public Builder input(final String input) {
+ if (null != input) {
+ this.input = input;
+ }
+ return this;
+ }
+
+ /**
+ * Defaults to empty string
+ *
+ * @param explanation of validation result
+ * @return the builder
+ */
+ public Builder explanation(final String explanation) {
+ if (null != explanation) {
+ this.explanation = explanation;
+ }
+ return this;
+ }
+
+ /**
+ * Defaults to empty string
+ *
+ * @param subject the thing that was validated
+ * @return the builder
+ */
+ public Builder subject(final String subject) {
+ if (null != subject) {
+ this.subject = subject;
+ }
+ return this;
+ }
+
+ public ValidationResult build() {
+ return new ValidationResult(this);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/components/Validator.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/components/Validator.java b/nifi-api/src/main/java/org/apache/nifi/components/Validator.java
new file mode 100644
index 0000000..a12b532
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/components/Validator.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.components;
+
+/**
+ *
+ */
+public interface Validator {
+
+ /**
+ * Validator object providing validation behavior in which validation always
+ * fails
+ */
+ Validator INVALID = new Validator() {
+ @Override
+ public ValidationResult validate(final String subject, final String input, final ValidationContext context) {
+ return new ValidationResult.Builder().subject(subject).explanation(String.format("'%s' is not a supported property", subject)).input(input).build();
+ }
+ };
+
+ /**
+ * Validator object providing validation behavior in which validation always
+ * passes
+ */
+ Validator VALID = new Validator() {
+ @Override
+ public ValidationResult validate(final String subject, final String input, final ValidationContext context) {
+ return new ValidationResult.Builder().subject(subject).input(input).valid(true).build();
+ }
+ };
+
+ /**
+ * @param subject what is being validated
+ * @param input the string to be validated
+ * @param context the ValidationContext to use when validating properties
+ * @return ValidationResult
+ * @throws NullPointerException of given input is null
+ */
+ ValidationResult validate(String subject, String input, ValidationContext context);
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/controller/AbstractControllerService.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/AbstractControllerService.java b/nifi-api/src/main/java/org/apache/nifi/controller/AbstractControllerService.java
new file mode 100644
index 0000000..cd3b9bd
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/AbstractControllerService.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller;
+
+import java.util.Map;
+
+import org.apache.nifi.components.AbstractConfigurableComponent;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.PropertyValue;
+import org.apache.nifi.controller.annotation.OnConfigured;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+import org.apache.nifi.reporting.InitializationException;
+
+public abstract class AbstractControllerService extends AbstractConfigurableComponent implements ControllerService {
+
+ private String identifier;
+ private ControllerServiceLookup serviceLookup;
+ private volatile ConfigurationContext configContext;
+ private ComponentLog logger;
+
+ @Override
+ public final void initialize(final ControllerServiceInitializationContext context) throws InitializationException {
+ this.identifier = context.getIdentifier();
+ serviceLookup = context.getControllerServiceLookup();
+ logger = context.getLogger();
+ init(context);
+ }
+
+ @Override
+ public String getIdentifier() {
+ return identifier;
+ }
+
+ @OnConfigured
+ public void onConfigurationChange(final ConfigurationContext context) {
+ this.configContext = context;
+ }
+
+ /**
+ * @param descriptor to retrieve value of
+ * @return the currently configured value for the given
+ * {@link PropertyDescriptor}
+ */
+ protected final PropertyValue getProperty(final PropertyDescriptor descriptor) {
+ return configContext.getProperty(descriptor);
+ }
+
+ /**
+ * @return an unmodifiable map of all configured properties for this
+ * {@link ControllerService}
+ */
+ protected final Map<PropertyDescriptor, String> getProperties() {
+ return configContext.getProperties();
+ }
+
+ /**
+ * @return the {@link ControllerServiceLookup} that was passed to the
+ * {@link #init(ProcessorInitializationContext)} method
+ */
+ protected final ControllerServiceLookup getControllerServiceLookup() {
+ return serviceLookup;
+ }
+
+ /**
+ * Provides a mechanism by which subclasses can perform initialization of
+ * the Reporting Task before it is scheduled to be run
+ *
+ * @param config of initialization context
+ * @throws InitializationException if unable to init
+ */
+ protected void init(final ControllerServiceInitializationContext config) throws InitializationException {
+ }
+
+ /**
+ * @return the logger that has been provided to the component by the
+ * framework in its initialize method
+ */
+ protected ComponentLog getLogger() {
+ return logger;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/controller/ConfigurationContext.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/ConfigurationContext.java b/nifi-api/src/main/java/org/apache/nifi/controller/ConfigurationContext.java
new file mode 100644
index 0000000..03965d4
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/ConfigurationContext.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller;
+
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.PropertyValue;
+
+/**
+ * This context is passed to ControllerServices and Reporting Tasks in order
+ * to expose their configuration to them.
+ */
+public interface ConfigurationContext {
+
+ /**
+ * @param property to retrieve by name
+ * @return the configured value for the property with the given name
+ */
+ PropertyValue getProperty(PropertyDescriptor property);
+
+ /**
+ * @return an unmodifiable map of all configured properties for this
+ * {@link ControllerService}
+ */
+ Map<PropertyDescriptor, String> getProperties();
+
+ /**
+ * @return a String representation of the scheduling period, or <code>null</code> if
+ * the component does not have a scheduling period (e.g., for ControllerServices)
+ */
+ String getSchedulingPeriod();
+
+ /**
+ * Returns the amount of time, in the given {@link TimeUnit} that will
+ * elapsed between the return of one execution of the
+ * component's <code>onTrigger</code> method and
+ * the time at which the method is invoked again. This method will return
+ * null if the component does not have a scheduling period (e.g., for ControllerServices)
+ *
+ * @param timeUnit unit of time for scheduling
+ * @return period of time or <code>null</code> if component does not have a scheduling
+ * period
+ */
+ Long getSchedulingPeriod(TimeUnit timeUnit);
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/controller/ControllerService.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/ControllerService.java b/nifi-api/src/main/java/org/apache/nifi/controller/ControllerService.java
new file mode 100644
index 0000000..a77c69d
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/ControllerService.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller;
+
+import org.apache.nifi.components.ConfigurableComponent;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.PropertyValue;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSessionFactory;
+import org.apache.nifi.processor.Processor;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.reporting.ReportingTask;
+
+/**
+ * <p>
+ * This interface provides a mechanism for creating services that are shared
+ * among all {@link Processor}s, {@link ReportingTask}s, and other
+ * {@code ControllerService}s.
+ * </p>
+ *
+ * <p>
+ * <code>ControllerService</code>s are discovered using Java's
+ * <code>ServiceLoader</code> mechanism. As a result, all implementations must
+ * follow these rules:
+ *
+ * <ul>
+ * <li>The implementation must implement this interface.</li>
+ * <li>The implementation must have a file named
+ * org.apache.nifi.controller.ControllerService located within the jar's
+ * <code>META-INF/services</code> directory. This file contains a list of
+ * fully-qualified class names of all <code>ControllerService</code>s in the
+ * jar, one-per-line.
+ * <li>The implementation must support a default constructor.</li>
+ * </ul>
+ * </p>
+ *
+ * <p>
+ * <b>All implementations of this interface must be thread-safe.</b>
+ * </p>
+ *
+ * <h2>Accessing Controller Services</h2>
+ * <p>
+ * A ControllerService is accessible only through its interface. The framework
+ * provides access to a ControllerService through two different mechanisms:
+ * <ul>
+ * <li>
+ * A {@link PropertyDescriptor} can be created via the
+ * {@link PropertyDescriptor.Builder} after calling the
+ * {@link PropertyDescriptor.Builder#identifiesControllerService(Class) identifiesControllerService(Class)}
+ * method and then the ControllerService is accessed via
+ * {@link PropertyValue#asControllerService(Class)} method.
+ * <p>
+ * For example:
+ * </p>
+ * <p>
+ * <code><pre>
+ * public static final PropertyDescriptor MY_PROPERTY = new PropertyDescriptor.Builder()
+ * .name("My Property")
+ * .description("Example Property")
+ * .identifiesControllerService( MyControllerServiceInterface.class )
+ * .build();
+ *
+ * ...
+ * public void onTrigger(ProcessContext context, ProcessSessionFactory sessionFactory) throws ProcessException {
+ * // Obtain the user-selected controller service
+ * final MyControllerServiceInterface service = context.getProperty(MY_PROPERTY).asControllerService( MyControllerServiceInterface.class );
+ * ...
+ * }
+ *
+ * </pre></code></p>
+ * </li>
+ * <li>A Controller Service can be obtained via a
+ * {@link ControllerServiceLookup}. This lookup may be obtained, for example,
+ * from the {@link ProcessContext} that is provided to a {@link Processor}'s
+ * {@link Processor#onTrigger(ProcessContext, ProcessSessionFactory) onTrigger}
+ * method.
+ * <p>
+ * For example:
+ * </p>
+ * <p>
+ * <code><pre>
+ * public void onTrigger(ProcessContext context, ProcessSessionFactory sessionFactory) throws ProcessException {
+ * final MyControllerServiceInterface service = (MyControllerServiceInterface) context.getControllerServiceLookup().getControllerService("service_identifier");
+ * }
+ * </pre></code></p>
+ * </li>
+ * </ul>
+ * </p>
+ *
+ * <h2>Defining a Controller Service</h2>
+ * <p>
+ * Note in both of the examples above, that the Controller Service was accessed
+ * only by its interface, and this interface extends ControllerService. If we
+ * have an implementation named MyServiceImpl, for example, that implements
+ * MyControllerServiceInterface, we cannot, in either case, attempt to cast the
+ * ControllerService to the desired implementation. Doing so will result in a
+ * {@link ClassCastException}. This is by design and is done for the following
+ * reasons:
+ *
+ * <ul>
+ * <li>
+ * It is a good coding practice to implement such a service as an interface in
+ * general.
+ * </li>
+ * <li>
+ * A Controller Service can be referenced from different NiFi Archives (NARs).
+ * This means that the Controller Service may be defined in one ClassLoader and
+ * referenced from another unrelated ClassLoader. In order to account for this,
+ * NiFi will change the current thread's ClassLoader as appropriate when
+ * entering the Controller Service's code and revert back to the previous
+ * ClassLoader after exiting the Controller Service's code.
+ * </li>
+ * </ul>
+ * </p>
+ *
+ * <h2>Controller Services and NARs</h2>
+ * <p>
+ * Due to the fact that a Controller Service may be referenced from a different
+ * NAR than the one in which the implementation lives, it is crucial that both
+ * the Controller Service's interface and the code referencing the interface
+ * inherit from the same ClassLoader. This is accomplished by ensuring that the
+ * NAR that contains the Controller Service interface is the parent (or
+ * ancestor) of the NAR that references the Controller Service interface.
+ * </p>
+ *
+ * <p>
+ * Typically, this is done by creating a NAR structure as follows:
+ * <pre>
+ * + my-services-api-nar
+ * +--- service-X-implementation-nar
+ * +--- service-Y-implementation-nar
+ * +--- service-Z-implementation-nar
+ * +--- processor-A-nar
+ * +--- processor-B-nar
+ * </pre>
+ * </p>
+ *
+ * <p>
+ * In this case, the {@code MyControllerServiceInterface} interface, and any
+ * other Controller Service interfaces, will be defined in the
+ * {@code my-services-api-nar} NAR. Implementations are then encapsulated within
+ * the {@code service-X-implementation-nar},
+ * {@code service-Y-implementation-nar}, and
+ * {@code service-Z-implementation-nar} NARs. All Controller Services and all
+ * Processors defined in these NARs are able to reference any other Controller
+ * Services whose interfaces are provided in the {@code my-services-api-nar}
+ * NAR.
+ * </p>
+ *
+ * <p>
+ * For more information on NARs, see the NiFi Developer Guide.
+ * </p>
+ */
+public interface ControllerService extends ConfigurableComponent {
+
+ /**
+ * Provides the Controller Service with access to objects that may be of use
+ * throughout the life of the service. This method will be called before any
+ * properties are set
+ *
+ * @param context of initialization
+ * @throws org.apache.nifi.reporting.InitializationException if unable to init
+ */
+ void initialize(ControllerServiceInitializationContext context) throws InitializationException;
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/controller/ControllerServiceInitializationContext.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/ControllerServiceInitializationContext.java b/nifi-api/src/main/java/org/apache/nifi/controller/ControllerServiceInitializationContext.java
new file mode 100644
index 0000000..6fcee0c
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/ControllerServiceInitializationContext.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller;
+
+import org.apache.nifi.logging.ComponentLog;
+
+public interface ControllerServiceInitializationContext {
+
+ /**
+ * @return the identifier associated with the {@link ControllerService} with
+ * which this context is associated
+ */
+ String getIdentifier();
+
+ /**
+ * @return the {@link ControllerServiceLookup} which can be used to obtain
+ * Controller Services
+ */
+ ControllerServiceLookup getControllerServiceLookup();
+
+ /**
+ * @return a logger that can be used to log important events in a standard
+ * way and generate bulletins when appropriate
+ */
+ ComponentLog getLogger();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/controller/ControllerServiceLookup.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/ControllerServiceLookup.java b/nifi-api/src/main/java/org/apache/nifi/controller/ControllerServiceLookup.java
new file mode 100644
index 0000000..f5300b1
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/ControllerServiceLookup.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller;
+
+import java.util.Set;
+
+public interface ControllerServiceLookup {
+
+ /**
+ * @param serviceIdentifier of controller service
+ * @return the ControllerService that is registered with the given
+ * identifier
+ */
+ ControllerService getControllerService(String serviceIdentifier);
+
+ /**
+ * @param serviceIdentifier identifier of service to check
+ * @return <code>true</code> if the Controller Service with the given
+ * identifier is enabled, <code>false</code> otherwise. If the given
+ * identifier is not known by this ControllerServiceLookup, returns
+ * <code>false</code>
+ */
+ boolean isControllerServiceEnabled(String serviceIdentifier);
+
+ /**
+ * @param serviceIdentifier idenfitier of service to check
+ * @return <code>true</code> if the Controller Service with the given
+ * identifier has been enabled but is still in the transitioning state,
+ * otherwise returns <code>false</code>. If the given identifier is not
+ * known by this ControllerServiceLookup, returns <code>false</code>
+ */
+ boolean isControllerServiceEnabling(String serviceIdentifier);
+
+ /**
+ * @param service service to check
+ * @return <code>true</code> if the given Controller Service is enabled,
+ * <code>false</code> otherwise. If the given Controller Service is not
+ * known by this ControllerServiceLookup, returns <code>false</code>
+ */
+ boolean isControllerServiceEnabled(ControllerService service);
+
+ /**
+ * @param serviceType type of service to get identifiers for
+ * @return the set of all Controller Service Identifiers whose Controller
+ * Service is of the given type.
+ * @throws IllegalArgumentException if the given class is not an interface
+ */
+ Set<String> getControllerServiceIdentifiers(Class<? extends ControllerService> serviceType) throws IllegalArgumentException;
+
+ /**
+ * @param serviceIdentifier identifier to look up
+ * @return the name of the Controller service with the given identifier. If
+ * no service can be found with this identifier, returns {@code null}
+ */
+ String getControllerServiceName(String serviceIdentifier);
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/controller/FlowFileQueue.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/FlowFileQueue.java b/nifi-api/src/main/java/org/apache/nifi/controller/FlowFileQueue.java
new file mode 100644
index 0000000..e1baeb7
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/FlowFileQueue.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Queue;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.nifi.controller.repository.FlowFileRecord;
+import org.apache.nifi.flowfile.FlowFilePrioritizer;
+import org.apache.nifi.processor.FlowFileFilter;
+import org.apache.nifi.processor.QueueSize;
+
+public interface FlowFileQueue {
+
+ /**
+ * @return the unique identifier for this FlowFileQueue
+ */
+ String getIdentifier();
+
+ /**
+ * @return list of processing priorities for this queue
+ */
+ List<FlowFilePrioritizer> getPriorities();
+
+ /**
+ * @return the minimum number of FlowFiles that must be present in order for
+ * FlowFiles to begin being swapped out of the queue
+ */
+ int getSwapThreshold();
+
+ /**
+ * Resets the comparator used by this queue to maintain order.
+ *
+ * @param newPriorities the ordered list of prioritizers to use to determine
+ * order within this queue.
+ * @throws NullPointerException if arg is null
+ */
+ void setPriorities(List<FlowFilePrioritizer> newPriorities);
+
+ /**
+ * Establishes this queue's preferred maximum work load.
+ *
+ * @param maxQueueSize the maximum number of flow files this processor
+ * recommends having in its work queue at any one time
+ */
+ void setBackPressureObjectThreshold(long maxQueueSize);
+
+ /**
+ * @return maximum number of flow files that should be queued up at any one
+ * time
+ */
+ long getBackPressureObjectThreshold();
+
+ /**
+ * @param maxDataSize Establishes this queue's preferred maximum data size.
+ */
+ void setBackPressureDataSizeThreshold(String maxDataSize);
+
+ /**
+ * @return maximum data size that should be queued up at any one time
+ */
+ String getBackPressureDataSizeThreshold();
+
+ QueueSize size();
+
+ /**
+ * @return total size in bytes of the queue flow file's content
+ */
+ long contentSize();
+
+ /**
+ * @return true if no items queue; false otherwise
+ */
+ boolean isEmpty();
+
+ /**
+ * @return true if the active queue is empty; false otherwise. The Active
+ * queue contains those FlowFiles that can be processed immediately and does
+ * not include those FlowFiles that have been swapped out or are currently
+ * being processed
+ */
+ boolean isActiveQueueEmpty();
+
+ QueueSize getActiveQueueSize();
+
+ /**
+ * Returns a QueueSize that represents all FlowFiles that are 'unacknowledged'. A FlowFile
+ * is considered to be unacknowledged if it has been pulled from the queue by some component
+ * but the session that pulled the FlowFile has not yet been committed or rolled back.
+ *
+ * @return a QueueSize that represents all FlowFiles that are 'unacknowledged'.
+ */
+ QueueSize getUnacknowledgedQueueSize();
+
+ void acknowledge(FlowFileRecord flowFile);
+
+ void acknowledge(Collection<FlowFileRecord> flowFiles);
+
+ /**
+ * @return true if maximum queue size has been reached or exceeded; false
+ * otherwise
+ */
+ boolean isFull();
+
+ /**
+ * places the given file into the queue
+ *
+ * @param file to place into queue
+ */
+ void put(FlowFileRecord file);
+
+ /**
+ * places the given files into the queue
+ *
+ * @param files to place into queue
+ */
+ void putAll(Collection<FlowFileRecord> files);
+
+ /**
+ * Removes all records from the internal swap queue and returns them.
+ *
+ * @return all removed records from internal swap queue
+ */
+ List<FlowFileRecord> pollSwappableRecords();
+
+ /**
+ * Restores the records from swap space into this queue, adding the records
+ * that have expired to the given set instead of enqueuing them.
+ *
+ * @param records that were swapped in
+ */
+ void putSwappedRecords(Collection<FlowFileRecord> records);
+
+ /**
+ * Updates the internal counters of how much data is queued, based on
+ * swapped data that is being restored.
+ *
+ * @param numRecords count of records swapped in
+ * @param contentSize total size of records being swapped in
+ */
+ void incrementSwapCount(int numRecords, long contentSize);
+
+ /**
+ * @return the number of FlowFiles that are enqueued and not swapped
+ */
+ int unswappedSize();
+
+ int getSwapRecordCount();
+
+ int getSwapQueueSize();
+
+ /**
+ * @param expiredRecords expired records
+ * @return the next flow file on the queue; null if empty
+ */
+ FlowFileRecord poll(Set<FlowFileRecord> expiredRecords);
+
+ /**
+ * @param maxResults limits how many results can be polled
+ * @param expiredRecords for expired records
+ * @return the next flow files on the queue up to the max results; null if
+ * empty
+ */
+ List<FlowFileRecord> poll(int maxResults, Set<FlowFileRecord> expiredRecords);
+
+ /**
+ * Drains flow files from the given source queue into the given destination
+ * list.
+ *
+ * @param sourceQueue queue to drain from
+ * @param destination Collection to drain to
+ * @param maxResults max number to drain
+ * @param expiredRecords for expired records
+ * @return size (bytes) of flow files drained from queue
+ */
+ long drainQueue(Queue<FlowFileRecord> sourceQueue, List<FlowFileRecord> destination, int maxResults, Set<FlowFileRecord> expiredRecords);
+
+ List<FlowFileRecord> poll(FlowFileFilter filter, Set<FlowFileRecord> expiredRecords);
+
+ String getFlowFileExpiration();
+
+ int getFlowFileExpiration(TimeUnit timeUnit);
+
+ void setFlowFileExpiration(String flowExpirationPeriod);
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/controller/ScheduledState.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/ScheduledState.java b/nifi-api/src/main/java/org/apache/nifi/controller/ScheduledState.java
new file mode 100644
index 0000000..d482eae
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/ScheduledState.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller;
+
+/**
+ * Indicates the valid values for the state of a <code>Triggerable</code> entity
+ * with respect to scheduling the entity to run.
+ */
+public enum ScheduledState {
+
+ /**
+ * Entity cannot be scheduled to run
+ */
+ DISABLED,
+ /**
+ * Entity can be scheduled to run but currently is not
+ */
+ STOPPED,
+ /**
+ * Entity is currently scheduled to run
+ */
+ RUNNING;
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/controller/Snippet.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/Snippet.java b/nifi-api/src/main/java/org/apache/nifi/controller/Snippet.java
new file mode 100644
index 0000000..93f3327
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/Snippet.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller;
+
+import java.util.Set;
+
+/**
+ * A Snippet represents a segment of the flow
+ */
+public interface Snippet {
+
+ /**
+ * @return id of this snippet
+ */
+ public String getId();
+
+ /**
+ * @return Whether or not this snippet is linked to the data flow. If the Snippet is
+ * deleted and is linked, then the underlying components will also be
+ * deleted. If the Snippet is deleted and is NOT linked, only the Snippet is
+ * removed
+ */
+ public boolean isLinked();
+
+ /**
+ * @return parent group id of the components in this snippet
+ */
+ public String getParentGroupId();
+
+ /**
+ * @return connections in this snippet
+ */
+ public Set<String> getConnections();
+
+ /**
+ * @return funnels in this snippet
+ */
+ public Set<String> getFunnels();
+
+ /**
+ * @return input ports in this snippet
+ */
+ public Set<String> getInputPorts();
+
+ /**
+ * @return output ports in this snippet
+ */
+ public Set<String> getOutputPorts();
+
+ /**
+ * @return labels in this snippet
+ */
+ public Set<String> getLabels();
+
+ /**
+ * @return the identifiers of all ProcessGroups in this Snippet
+ */
+ public Set<String> getProcessGroups();
+
+ /**
+ * @return the identifiers of all Processors in this Snippet
+ */
+ public Set<String> getProcessors();
+
+ /**
+ * @return the identifiers of all RemoteProcessGroups in this Snippet
+ */
+ public Set<String> getRemoteProcessGroups();
+
+ /**
+ * @return Determines if this snippet is empty
+ */
+ public boolean isEmpty();
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/controller/Triggerable.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/Triggerable.java b/nifi-api/src/main/java/org/apache/nifi/controller/Triggerable.java
new file mode 100644
index 0000000..4b3149b
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/Triggerable.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.ProcessSessionFactory;
+import org.apache.nifi.processor.exception.ProcessException;
+
+public interface Triggerable {
+
+ public static final long MINIMUM_SCHEDULING_NANOS = 30000L;
+
+ /**
+ * <p>
+ * The method called when this processor is triggered to operate by the
+ * controller. This method may be called concurrently from different
+ * threads. When this method is called depends on how this processor is
+ * configured within a controller to be triggered (timing or event
+ * based).</p>
+ *
+ * <p>
+ * The processor may commit, roll back, or allow the framework to
+ * automatically manage the session. If the sessions are to be managed by
+ * the framework (recommended) then what it will do depends on several
+ * factors. If the method call returns due to an exception then the session
+ * will be rolled back. If the method returns normally then the session will
+ * be committed or the framework may use the session again for another
+ * processor down stream</p>
+ *
+ * @param context in which the component is triggered
+ * @param sessionFactory used to generate {@link ProcessSession}s to use for
+ * operating on flow files within the repository
+ *
+ * @throws ProcessException if processing did not complete normally though
+ * indicates the problem is an understood potential outcome of processing.
+ * The controller/caller will handle these exceptions gracefully such as
+ * logging, etc.. If another type of exception is allowed to propagate the
+ * controller may no longer trigger this processor to operate as this would
+ * indicate a probable coding defect.
+ */
+ void onTrigger(ProcessContext context, ProcessSessionFactory sessionFactory) throws ProcessException;
+
+ /**
+ * Determines the number of concurrent tasks that may be running for this
+ * <code>Triggerable</code>.
+ *
+ * @param taskCount a number of concurrent tasks this processor may have
+ * running
+ * @throws IllegalArgumentException if the given value is less than 1
+ */
+ void setMaxConcurrentTasks(int taskCount);
+
+ /**
+ * @return the number of tasks that may execute concurrently for this
+ * <code>Triggerable</code>.
+ */
+ int getMaxConcurrentTasks();
+
+ /**
+ * Indicates the {@link ScheduledState} of this <code>Triggerable</code>. A
+ * value of stopped does NOT indicate that the <code>Triggerable</code> has
+ * no active threads, only that it is not currently scheduled to be given
+ * any more threads. To determine whether or not the
+ * <code>Triggerable</code> has any active threads, see
+ * {@link ProcessScheduler#getActiveThreadCount(nifi.connectable.Connectable)}.
+ *
+ * @return the schedule state
+ */
+ ScheduledState getScheduledState();
+
+ /**
+ * Indicates whether or not this <code>Triggerable</code> is "running". It
+ * is considered "running" if it is scheduled to run OR if it is no longer
+ * scheduled to be given threads but the remaining threads from the last
+ * invocation of {@link #onTrigger(ProcessContext, ProcessSessionFactory)}
+ * have not yet returned
+ *
+ * @return true if running;false otherwise
+ */
+ boolean isRunning();
+
+ /**
+ * @param timeUnit for the scheduling period of the component
+ * @return the amount of time between each scheduling period
+ */
+ long getSchedulingPeriod(TimeUnit timeUnit);
+
+ /**
+ * @return a string representation of the time between each scheduling
+ * period
+ */
+ String getSchedulingPeriod();
+
+ /**
+ * Updates how often this Triggerable should be triggered to run
+ *
+ * @param schedulingPeriod to set
+ */
+ void setScheduldingPeriod(String schedulingPeriod);
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/controller/annotation/OnConfigured.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/annotation/OnConfigured.java b/nifi-api/src/main/java/org/apache/nifi/controller/annotation/OnConfigured.java
new file mode 100644
index 0000000..aef80ac
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/annotation/OnConfigured.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller.annotation;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Marker annotation a ControllerService implementation can use to indicate a
+ * method should be called after all of the properties have been set for the
+ * Controller Service. Methods using this annotation must take either 0
+ * arguments or a single argument of type
+ * {@link nifi.controller.ConfigurationContext ConfigurationContext}.
+ *
+ *
+ * @deprecated This annotation has been replaced by those in the
+ * {@link org.apache.nifi.annotation.lifecycle} package.
+ */
+@Documented
+@Target({ElementType.METHOD})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+@Deprecated
+public @interface OnConfigured {
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/controller/repository/ContentRepository.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/repository/ContentRepository.java b/nifi-api/src/main/java/org/apache/nifi/controller/repository/ContentRepository.java
new file mode 100644
index 0000000..ee3ead9
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/repository/ContentRepository.java
@@ -0,0 +1,303 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller.repository;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Path;
+import java.util.Collection;
+import java.util.Set;
+
+import org.apache.nifi.controller.repository.claim.ContentClaim;
+import org.apache.nifi.controller.repository.claim.ContentClaimManager;
+
+/**
+ * Defines the capabilities of a content repository. Append options are not
+ * available on the methods but a merge capability is provided which between
+ * that and creating new claims a merge is available.
+ *
+ */
+public interface ContentRepository {
+
+ /**
+ * Initializes the Content Repository, providing to it the
+ * ContentClaimManager that is to be used for interacting with Content
+ * Claims
+ *
+ * @param claimManager to handle claims
+ * @throws java.io.IOException if unable to init
+ */
+ void initialize(ContentClaimManager claimManager) throws IOException;
+
+ /**
+ * Shuts down the Content Repository, freeing any resources that may be
+ * held. This is called when an administrator shuts down NiFi.
+ */
+ void shutdown();
+
+ /**
+ * @return the names of all Containers that exist for this Content
+ * Repository
+ */
+ Set<String> getContainerNames();
+
+ /**
+ * @param containerName name of container to check capacity on
+ * @return the maximum number of bytes that can be stored in the storage
+ * mechanism that backs the container with the given name
+ * @throws java.io.IOException if unable to check capacity
+ * @throws IllegalArgumentException if no container exists with the given
+ * name
+ */
+ long getContainerCapacity(String containerName) throws IOException;
+
+ /**
+ * @param containerName to check space on
+ * @return the number of bytes available to be used used by the storage
+ * mechanism that backs the container with the given name
+ * @throws java.io.IOException if unable to check space
+ * @throws IllegalArgumentException if no container exists with the given
+ * name
+ */
+ long getContainerUsableSpace(String containerName) throws IOException;
+
+ /**
+ * Creates a new content claim
+ *
+ * @param lossTolerant indicates whether the content for the new claim is
+ * loss tolerant. If true the repository might choose more volatile storage
+ * options which could increase performance for a tradeoff with reliability
+ * @return newly created claim
+ * @throws java.io.IOException if unable to create claim
+ */
+ ContentClaim create(boolean lossTolerant) throws IOException;
+
+ /**
+ * Increments the number of claimants for the given claim
+ *
+ * @param claim to increment
+ * @return the number of claimants after incrementing
+ */
+ int incrementClaimaintCount(ContentClaim claim);
+
+ /**
+ * Obtains the current number of claimants for the given claim
+ *
+ * @param claim to get count of
+ * @return the number of claimants
+ */
+ int getClaimantCount(ContentClaim claim);
+
+ /**
+ * Reduces the number of claimants for the given claim. Even if the given
+ * claim is null or content cannot be found or removed no exception will be
+ * thrown.
+ *
+ * @param claim to decrement
+ * @return new claimant count for the given claim
+ */
+ int decrementClaimantCount(ContentClaim claim);
+
+ /**
+ * Removes the content indicated by the given claim
+ *
+ * @param claim to remove
+ *
+ * @return a boolean indicating whether or not the destruction of the claim
+ * was successful
+ */
+ boolean remove(ContentClaim claim);
+
+ /**
+ * Clones the content for the given content claim and returns content claim
+ * of the new object
+ *
+ * @param original to clone
+ * @param lossTolerant if can be place in a loss tolerant repository
+ * @return new claim
+ * @throws IOException if an IO error occurs. Any content written to the new
+ * destination prior to the error will be destroyed
+ */
+ ContentClaim clone(ContentClaim original, boolean lossTolerant) throws IOException;
+
+ /**
+ * Creates a new content item that is the merger in iteration order of all
+ * content for the given claims
+ *
+ * @return the size of the destination
+ * @param claims the claims to merge which will be combined in order of
+ * collection iteration
+ * @param destination the claim to write the merged content to
+ * @param header if supplied will be prepended to the output
+ * @param footer if supplied will be appended to the output
+ * @param demarcator if supplied will be placed in between each merged
+ * object
+ * @throws IOException if unable to merge
+ * @throws IllegalArgumentException if the given destination is included in
+ * the given claims
+ */
+ long merge(Collection<ContentClaim> claims, ContentClaim destination, byte[] header, byte[] footer, byte[] demarcator) throws IOException;
+
+ /**
+ * Imports content from the given path creating a new content object and
+ * claim within the repository.
+ *
+ * @return the size of the claim
+ * @param content to import from
+ * @param claim the claim to write imported content to
+ * @throws IOException if failure to read given content
+ */
+ long importFrom(Path content, ContentClaim claim) throws IOException;
+
+ /**
+ * Imports content from the given path to the specified claim, appending or
+ * replacing the current claim, according to the value of the append
+ * argument
+ *
+ * @return the size of the claim
+ * @param content to import from
+ * @param claim the claim to write imported content to
+ * @param append if true, the content will be appended to the claim; if
+ * false, the content will replace the contents of the claim
+ * @throws IOException if unable to read content
+ */
+ long importFrom(Path content, ContentClaim claim, boolean append) throws IOException;
+
+ /**
+ * Imports content from the given stream creating a new content object and
+ * claim within the repository.
+ *
+ * @return the size of the claim
+ * @param content to import from
+ * @param claim the claim to write imported content to
+ * @throws IOException if unable to read content
+ */
+ long importFrom(InputStream content, ContentClaim claim) throws IOException;
+
+ /**
+ * Imports content from the given stream, appending or replacing the current
+ * claim, according to the value of the appen dargument
+ *
+ * @param content to import from
+ * @param claim to write to
+ * @param append whether to append or replace
+ * @return length of data imported in bytes
+ * @throws IOException if failure to read or write stream
+ */
+ long importFrom(InputStream content, ContentClaim claim, boolean append) throws IOException;
+
+ /**
+ * Exports the content of the given claim to the given destination.
+ *
+ * @return the size of the destination or the claim
+ * @param claim to export from
+ * @param destination where to export data
+ * @param append if true appends to the destination; false overwrites
+ * @throws IOException if an IO error occurs. The state of the content for
+ * the given destination is unknown and callers should consider whether they
+ * should clean up any partially created paths
+ */
+ long exportTo(ContentClaim claim, Path destination, boolean append) throws IOException;
+
+ /**
+ * Exports the content of the given claim to the given destination.
+ *
+ * @return the size of the destination or the claim
+ * @param claim to export from
+ * @param destination where to export data
+ * @param append if true appends to the destination; false overwrites
+ * @param offset the offset at which the claim should start being copied
+ * @param length the number of bytes to copy
+ * @throws IOException if an IO error occurs. The state of the content for
+ * the given destination is unknown and callers should consider whether they
+ * should clean up any partially created paths
+ */
+ long exportTo(ContentClaim claim, Path destination, boolean append, long offset, long length) throws IOException;
+
+ /**
+ * Exports the content of the given claim to the given destination.
+ *
+ * @return the size of the claim
+ * @param claim to export from
+ * @param destination where to export data
+ * @throws IOException if an IO error occurs.
+ */
+ long exportTo(ContentClaim claim, OutputStream destination) throws IOException;
+
+ /**
+ * Exports a subset of the content of the given claim, starting at offset
+ * and copying length bytes, to the given destination.
+ *
+ * @return the number of bytes copied
+ * @param claim to export from
+ * @param destination where to export data
+ * @param offset the offset into the claim at which the copy should begin
+ * @param length the number of bytes to copy
+ * @throws IOException if an IO error occurs.
+ */
+ long exportTo(ContentClaim claim, OutputStream destination, long offset, long length) throws IOException;
+
+ /**
+ * @param claim to get size of
+ * @return size in bytes of content for given claim
+ * @throws IOException if size check failed
+ */
+ long size(ContentClaim claim) throws IOException;
+
+ /**
+ * Provides access to the input stream for the given claim
+ *
+ * @param claim to read from
+ * @return InputStream over the content of the given claim
+ * @throws IOException if unable to read
+ */
+ InputStream read(ContentClaim claim) throws IOException;
+
+ /**
+ * Obtains an OutputStream to the content for the given claim.
+ *
+ * @param claim to write to
+ * @return the stream to write to
+ * @throws IOException if unable to obtain stream
+ */
+ OutputStream write(ContentClaim claim) throws IOException;
+
+ /**
+ * Purges the contents of the repository, as if the repository were newly
+ * created.
+ */
+ void purge();
+
+ /**
+ * Performs any cleanup actions that may need to be taken upon system
+ * restart. For example, if content was partially written to the repository
+ * before the restart, the repository is given a chance to handle this data
+ */
+ void cleanup();
+
+ /**
+ * @param contentClaim the Content Claim to check
+ * @return Returns a boolean indicating whether or not the content specified
+ * by the given claim can be read, regardless of whether the content has
+ * been archived or not. If the ContentRepository does not implement
+ * archiving capabilities, this method will return <code>false</code>
+ *
+ * @throws IOException if unable to determine accessibility
+ */
+ boolean isAccessible(ContentClaim contentClaim) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/controller/repository/FlowFileRecord.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/repository/FlowFileRecord.java b/nifi-api/src/main/java/org/apache/nifi/controller/repository/FlowFileRecord.java
new file mode 100644
index 0000000..f2493f6
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/repository/FlowFileRecord.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller.repository;
+
+import org.apache.nifi.controller.repository.claim.ContentClaim;
+import org.apache.nifi.flowfile.FlowFile;
+
+/**
+ * <code>FlowFileRecord</code> is a sub-interface of <code>FlowFile</code> and
+ * is used to provide additional information about FlowFiles that provide
+ * valuable information to the framework but should be hidden from components
+ */
+public interface FlowFileRecord extends FlowFile {
+
+ /**
+ * @return the time (in millis since epoch) at which this FlowFile should no
+ * longer be penalized
+ */
+ long getPenaltyExpirationMillis();
+
+ /**
+ * @return the {@link ContentClaim} that holds the FlowFile's content
+ */
+ ContentClaim getContentClaim();
+
+ /**
+ * @return the byte offset into the {@link ContentClaim} at which the
+ * FlowFile's content occurs. This mechanism allows multiple FlowFiles to
+ * have the same ContentClaim, which can be significantly more efficient for
+ * some implementations of
+ * {@link nifi.controller.repository.ContentRepository ContentRepository}
+ */
+ long getContentClaimOffset();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/controller/repository/FlowFileRepository.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/repository/FlowFileRepository.java b/nifi-api/src/main/java/org/apache/nifi/controller/repository/FlowFileRepository.java
new file mode 100644
index 0000000..5e59e04
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/repository/FlowFileRepository.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller.repository;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.nifi.controller.FlowFileQueue;
+import org.apache.nifi.controller.repository.claim.ContentClaimManager;
+
+/**
+ * Implementations must be thread safe
+ *
+ */
+public interface FlowFileRepository extends Closeable {
+
+ /**
+ * Initializes the Content Repository, providing to it the
+ * ContentClaimManager that is to be used for interacting with Content
+ * Claims
+ *
+ * @param claimManager for handling claims
+ * @throws java.io.IOException if unable to initialize repository
+ */
+ void initialize(ContentClaimManager claimManager) throws IOException;
+
+ /**
+ * @return the maximum number of bytes that can be stored in the underlying
+ * storage mechanism
+ *
+ * @throws IOException if computing capacity fails
+ */
+ long getStorageCapacity() throws IOException;
+
+ /**
+ * @return the number of bytes currently available for use by the underlying
+ * storage mechanism
+ *
+ * @throws IOException if computing usable space fails
+ */
+ long getUsableStorageSpace() throws IOException;
+
+ /**
+ * Updates the repository with the given RepositoryRecords.
+ *
+ * @param records the records to update the repository with
+ * @throws java.io.IOException if update fails
+ */
+ void updateRepository(Collection<RepositoryRecord> records) throws IOException;
+
+ /**
+ * Loads all flow files found within the repository, establishes the content
+ * claims and their reference count
+ *
+ * @param queueProvider the provider of FlowFile Queues into which the
+ * FlowFiles should be enqueued
+ * @param minimumSequenceNumber specifies the minimum value that should be
+ * returned by a call to {@link #getNextFlowFileSequence()}
+ *
+ * @return index of highest flow file identifier
+ * @throws IOException if load fails
+ */
+ long loadFlowFiles(QueueProvider queueProvider, long minimumSequenceNumber) throws IOException;
+
+ /**
+ * @return <code>true</code> if the Repository is volatile (i.e., its data
+ * is lost upon application restart), <code>false</code> otherwise
+ */
+ boolean isVolatile();
+
+ /**
+ * @return the next ID in sequence for creating <code>FlowFile</code>s.
+ */
+ long getNextFlowFileSequence();
+
+ /**
+ * @return the max ID of all <code>FlowFile</code>s that currently exist in
+ * the repository.
+ * @throws IOException if computing max identifier fails
+ */
+ long getMaxFlowFileIdentifier() throws IOException;
+
+ /**
+ * Updates the Repository to indicate that the given FlowFileRecords were
+ * Swapped Out of memory
+ *
+ * @param swappedOut the FlowFiles that were swapped out of memory
+ * @param flowFileQueue the queue that the FlowFiles belong to
+ * @param swapLocation the location to which the FlowFiles were swapped
+ *
+ * @throws IOException if swap fails
+ */
+ void swapFlowFilesOut(List<FlowFileRecord> swappedOut, FlowFileQueue flowFileQueue, String swapLocation) throws IOException;
+
+ /**
+ * Updates the Repository to indicate that the given FlowFileRecpords were
+ * Swapped In to memory
+ *
+ * @param swapLocation the location (e.g., a filename) from which FlowFiles
+ * were recovered
+ * @param flowFileRecords the records that were swapped in
+ * @param flowFileQueue the queue that the FlowFiles belong to
+ *
+ * @throws IOException if swap fails
+ */
+ void swapFlowFilesIn(String swapLocation, List<FlowFileRecord> flowFileRecords, FlowFileQueue flowFileQueue) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/controller/repository/FlowFileSwapManager.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/repository/FlowFileSwapManager.java b/nifi-api/src/main/java/org/apache/nifi/controller/repository/FlowFileSwapManager.java
new file mode 100644
index 0000000..869e2b3
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/repository/FlowFileSwapManager.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller.repository;
+
+import org.apache.nifi.controller.repository.claim.ContentClaimManager;
+import org.apache.nifi.events.EventReporter;
+
+/**
+ * Defines a mechanism by which FlowFiles can be move into external storage or
+ * memory so that they can be removed from the Java heap and vice-versa
+ */
+public interface FlowFileSwapManager {
+
+ /**
+ * Starts the Manager's background threads to start swapping FlowFiles in
+ * and out of memory
+ *
+ * @param flowFileRepository the FlowFileRepository that must be notified of
+ * any swapping in or out of FlowFiles
+ * @param queueProvider the provider of FlowFileQueue's so that FlowFiles
+ * can be obtained and restored
+ * @param claimManager the ContentClaimManager to use for interacting with
+ * Content Claims
+ * @param reporter the EventReporter that can be used for notifying users of
+ * important events
+ */
+ void start(FlowFileRepository flowFileRepository, QueueProvider queueProvider, ContentClaimManager claimManager, EventReporter reporter);
+
+ /**
+ * Shuts down the manager
+ */
+ void shutdown();
+
+ /**
+ * Removes all Swap information, permanently destroying any FlowFiles that
+ * have been swapped out
+ */
+ void purge();
+
+ /**
+ * Notifies FlowFile queues of the number of FlowFiles and content size of
+ * all FlowFiles that are currently swapped out
+ *
+ * @param connectionProvider provider
+ * @param claimManager manager
+ * @return how many flowfiles have been recovered
+ */
+ long recoverSwappedFlowFiles(QueueProvider connectionProvider, ContentClaimManager claimManager);
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/controller/repository/QueueProvider.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/repository/QueueProvider.java b/nifi-api/src/main/java/org/apache/nifi/controller/repository/QueueProvider.java
new file mode 100644
index 0000000..fcb516d
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/repository/QueueProvider.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller.repository;
+
+import java.util.Collection;
+
+import org.apache.nifi.controller.FlowFileQueue;
+
+/**
+ * Provides a collection of <code>FlowFileQueue</code>s that represents all
+ * queues in the current flow
+ */
+public interface QueueProvider {
+
+ /**
+ * @return all <code>FlowFileQueue</code>s that currently exist in the flow
+ */
+ Collection<FlowFileQueue> getAllQueues();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/controller/repository/RepositoryRecord.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/repository/RepositoryRecord.java b/nifi-api/src/main/java/org/apache/nifi/controller/repository/RepositoryRecord.java
new file mode 100644
index 0000000..40d44a8
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/repository/RepositoryRecord.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller.repository;
+
+import org.apache.nifi.controller.FlowFileQueue;
+import org.apache.nifi.controller.repository.claim.ContentClaim;
+
+/**
+ * Represents an abstraction of a FlowFile that can be used to track changing
+ * state of a FlowFile so that transactionality with repositories can exist
+ */
+public interface RepositoryRecord {
+
+ /**
+ * @return FlowFileQueue to which the FlowFile is to be transferred
+ */
+ FlowFileQueue getDestination();
+
+ /**
+ * @return FlowFileQueue from which the record was pulled
+ */
+ FlowFileQueue getOriginalQueue();
+
+ /**
+ * @return type of update that this record encapsulates
+ */
+ RepositoryRecordType getType();
+
+ /**
+ * @return current ContentClaim for the FlowFile
+ */
+ ContentClaim getCurrentClaim();
+
+ /**
+ * @return original ContentClaim for the FlowFile before any changes were made
+ */
+ ContentClaim getOriginalClaim();
+
+ /**
+ * @return byte offset into the Content Claim where this FlowFile's content
+ * begins
+ */
+ long getCurrentClaimOffset();
+
+ /**
+ * @return FlowFile being encapsulated by this record
+ */
+ FlowFileRecord getCurrent();
+
+ /**
+ * @return Whether or not the FlowFile's attributes have changed since the FlowFile
+ * was pulled from its queue (or created)
+ */
+ boolean isAttributesChanged();
+
+ /**
+ * @return <code>true</code> if the FlowFile is not viable and should be
+ * aborted (e.g., the content of the FlowFile cannot be found)
+ */
+ boolean isMarkedForAbort();
+
+ /**
+ * @return If the FlowFile is swapped out of the Java heap space, provides the
+ * location of the swap file, or <code>null</code> if the FlowFile is not
+ * swapped out
+ */
+ String getSwapLocation();
+}
[02/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/AuthorizeUserActionTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/AuthorizeUserActionTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/AuthorizeUserActionTest.java
new file mode 100644
index 0000000..28ea4a9
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/AuthorizeUserActionTest.java
@@ -0,0 +1,433 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import java.util.Date;
+import java.util.EnumSet;
+import java.util.Set;
+import org.apache.nifi.admin.dao.AuthorityDAO;
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.admin.dao.DataAccessException;
+import org.apache.nifi.admin.dao.UserDAO;
+import org.apache.nifi.admin.service.AccountDisabledException;
+import org.apache.nifi.admin.service.AccountNotFoundException;
+import org.apache.nifi.admin.service.AccountPendingException;
+import org.apache.nifi.admin.service.AdministrationException;
+import org.apache.nifi.authorization.Authority;
+import org.apache.nifi.authorization.AuthorityProvider;
+import org.apache.nifi.authorization.exception.AuthorityAccessException;
+import org.apache.nifi.authorization.exception.UnknownIdentityException;
+import org.apache.nifi.user.AccountStatus;
+import org.apache.nifi.user.NiFiUser;
+import org.apache.commons.lang3.StringUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+/**
+ *
+ */
+public class AuthorizeUserActionTest {
+
+ private static final String USER_ID_6 = "6";
+ private static final String USER_ID_7 = "7";
+ private static final String USER_ID_8 = "8";
+ private static final String USER_ID_9 = "9";
+ private static final String USER_ID_10 = "10";
+ private static final String USER_ID_11 = "11";
+
+ private static final String USER_DN_1 = "authority access exception while searching for user";
+ private static final String USER_DN_2 = "unknown user";
+ private static final String USER_DN_3 = "user removed after checking existence";
+ private static final String USER_DN_4 = "access exception getting authorities";
+ private static final String USER_DN_5 = "error creating user account";
+ private static final String USER_DN_6 = "create user general sequence";
+ private static final String USER_DN_7 = "existing user requires verification";
+ private static final String USER_DN_8 = "existing user does not require verification";
+ private static final String USER_DN_9 = "existing pending user";
+ private static final String USER_DN_10 = "existing disabled user";
+ private static final String USER_DN_11 = "existing user is now unknown in the authority provider";
+
+ private DAOFactory daoFactory;
+ private UserDAO userDao;
+ private AuthorityDAO authorityDao;
+ private AuthorityProvider authorityProvider;
+
+ @Before
+ public void setup() throws Exception {
+ // mock the user dao
+ userDao = Mockito.mock(UserDAO.class);
+ Mockito.doAnswer(new Answer<NiFiUser>() {
+ @Override
+ public NiFiUser answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ String id = (String) args[0];
+
+ NiFiUser user = null;
+ if (USER_ID_7.equals(id)) {
+ user = new NiFiUser();
+ user.setId(USER_ID_7);
+ user.setDn(USER_DN_7);
+ user.getAuthorities().addAll(EnumSet.of(Authority.ROLE_MONITOR));
+ } else if (USER_ID_8.equals(id)) {
+ user = new NiFiUser();
+ user.setId(USER_ID_8);
+ user.setDn(USER_DN_8);
+ user.getAuthorities().addAll(EnumSet.of(Authority.ROLE_MONITOR));
+ user.setLastVerified(new Date());
+ } else if (USER_ID_11.equals(id)) {
+ user = new NiFiUser();
+ user.setId(USER_ID_11);
+ user.setDn(USER_DN_11);
+ user.getAuthorities().addAll(EnumSet.of(Authority.ROLE_MONITOR));
+ user.setStatus(AccountStatus.ACTIVE);
+ }
+
+ return user;
+ }
+ }).when(userDao).findUserById(Mockito.anyString());
+ Mockito.doAnswer(new Answer<NiFiUser>() {
+ @Override
+ public NiFiUser answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ String dn = (String) args[0];
+
+ NiFiUser user = null;
+ switch (dn) {
+ case USER_DN_7:
+ user = new NiFiUser();
+ user.setId(USER_ID_7);
+ user.setDn(USER_DN_7);
+ user.getAuthorities().addAll(EnumSet.of(Authority.ROLE_MONITOR));
+ break;
+ case USER_DN_8:
+ user = new NiFiUser();
+ user.setId(USER_ID_8);
+ user.setDn(USER_DN_8);
+ user.getAuthorities().addAll(EnumSet.of(Authority.ROLE_MONITOR));
+ user.setLastVerified(new Date());
+ break;
+ case USER_DN_9:
+ user = new NiFiUser();
+ user.setId(USER_ID_9);
+ user.setDn(USER_DN_9);
+ user.setStatus(AccountStatus.PENDING);
+ break;
+ case USER_DN_10:
+ user = new NiFiUser();
+ user.setId(USER_ID_10);
+ user.setDn(USER_DN_10);
+ user.setStatus(AccountStatus.DISABLED);
+ break;
+ case USER_DN_11:
+ user = new NiFiUser();
+ user.setId(USER_ID_11);
+ user.setDn(USER_DN_11);
+ user.getAuthorities().addAll(EnumSet.of(Authority.ROLE_MONITOR));
+ user.setStatus(AccountStatus.ACTIVE);
+ break;
+ }
+
+ return user;
+ }
+ }).when(userDao).findUserByDn(Mockito.anyString());
+ Mockito.doAnswer(new Answer<Void>() {
+ @Override
+ public Void answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ NiFiUser user = (NiFiUser) args[0];
+ switch (user.getDn()) {
+ case USER_DN_5:
+ throw new DataAccessException();
+ case USER_DN_6:
+ user.setId(USER_ID_6);
+ break;
+ }
+
+ // do nothing
+ return null;
+ }
+ }).when(userDao).createUser(Mockito.any(NiFiUser.class));
+ Mockito.doAnswer(new Answer<Void>() {
+ @Override
+ public Void answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ NiFiUser user = (NiFiUser) args[0];
+
+ // do nothing
+ return null;
+ }
+ }).when(userDao).updateUser(Mockito.any(NiFiUser.class));
+
+ // mock the authority dao
+ authorityDao = Mockito.mock(AuthorityDAO.class);
+ Mockito.doAnswer(new Answer<Void>() {
+ @Override
+ public Void answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ Set<Authority> authorities = (Set<Authority>) args[0];
+ String id = (String) args[1];
+
+ // do nothing
+ return null;
+ }
+ }).when(authorityDao).createAuthorities(Mockito.anySetOf(Authority.class), Mockito.anyString());
+ Mockito.doAnswer(new Answer<Void>() {
+ @Override
+ public Void answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ Set<Authority> authorities = (Set<Authority>) args[0];
+ String id = (String) args[1];
+
+ // do nothing
+ return null;
+ }
+ }).when(authorityDao).deleteAuthorities(Mockito.anySetOf(Authority.class), Mockito.anyString());
+
+ // mock the dao factory
+ daoFactory = Mockito.mock(DAOFactory.class);
+ Mockito.when(daoFactory.getUserDAO()).thenReturn(userDao);
+ Mockito.when(daoFactory.getAuthorityDAO()).thenReturn(authorityDao);
+
+ // mock the authority provider
+ authorityProvider = Mockito.mock(AuthorityProvider.class);
+ Mockito.doAnswer(new Answer<Boolean>() {
+ @Override
+ public Boolean answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ String dn = (String) args[0];
+ switch (dn) {
+ case USER_DN_1:
+ throw new AuthorityAccessException(StringUtils.EMPTY);
+ case USER_DN_2:
+ return false;
+ }
+
+ return true;
+ }
+ }).when(authorityProvider).doesDnExist(Mockito.anyString());
+ Mockito.doAnswer(new Answer<Set<Authority>>() {
+ @Override
+ public Set<Authority> answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ String dn = (String) args[0];
+ Set<Authority> authorities = EnumSet.noneOf(Authority.class);
+ switch (dn) {
+ case USER_DN_3:
+ throw new UnknownIdentityException(StringUtils.EMPTY);
+ case USER_DN_4:
+ throw new AuthorityAccessException(StringUtils.EMPTY);
+ case USER_DN_6:
+ authorities.add(Authority.ROLE_MONITOR);
+ break;
+ case USER_DN_7:
+ authorities.add(Authority.ROLE_DFM);
+ break;
+ case USER_DN_9:
+ throw new UnknownIdentityException(StringUtils.EMPTY);
+ case USER_DN_10:
+ throw new UnknownIdentityException(StringUtils.EMPTY);
+ case USER_DN_11:
+ throw new UnknownIdentityException(StringUtils.EMPTY);
+ }
+
+ return authorities;
+ }
+ }).when(authorityProvider).getAuthorities(Mockito.anyString());
+ Mockito.doAnswer(new Answer<Void>() {
+ @Override
+ public Void answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ String dn = (String) args[0];
+ Set<Authority> authorites = (Set<Authority>) args[1];
+
+ // do nothing
+ return null;
+ }
+ }).when(authorityProvider).setAuthorities(Mockito.anyString(), Mockito.anySet());
+ }
+
+ /**
+ * Tests AuthorityAccessException in doesDnExist.
+ *
+ * @throws Exception ex
+ */
+ @Test(expected = AdministrationException.class)
+ public void testAuthorityAccessExceptionInDoesDnExist() throws Exception {
+ AuthorizeUserAction authorizeUser = new AuthorizeUserAction(USER_DN_1, 0);
+ authorizeUser.execute(daoFactory, authorityProvider);
+ }
+
+ /**
+ * Test unknown user in the authority provider.
+ *
+ * @throws Exception ex
+ */
+ @Test(expected = AccountNotFoundException.class)
+ public void testUnknownUser() throws Exception {
+ AuthorizeUserAction authorizeUser = new AuthorizeUserAction(USER_DN_2, 0);
+ authorizeUser.execute(daoFactory, authorityProvider);
+ }
+
+ /**
+ * Test a user thats been removed after checking their existence.
+ *
+ * @throws Exception ex
+ */
+ @Test(expected = AccountNotFoundException.class)
+ public void testUserRemovedAfterCheckingExistence() throws Exception {
+ AuthorizeUserAction authorizeUser = new AuthorizeUserAction(USER_DN_3, 0);
+ authorizeUser.execute(daoFactory, authorityProvider);
+ }
+
+ /**
+ * Testing AuthorityAccessException when getting authorities.
+ *
+ * @throws Exception ex
+ */
+ @Test(expected = AdministrationException.class)
+ public void testAuthorityAccessException() throws Exception {
+ AuthorizeUserAction authorizeUser = new AuthorizeUserAction(USER_DN_4, 0);
+ authorizeUser.execute(daoFactory, authorityProvider);
+ }
+
+ /**
+ * Testing DataAccessException while creating user accounts.
+ *
+ * @throws Exception ex
+ */
+ @Test(expected = DataAccessException.class)
+ public void testErrorCreatingUserAccount() throws Exception {
+ AuthorizeUserAction authorizeUser = new AuthorizeUserAction(USER_DN_5, 0);
+ authorizeUser.execute(daoFactory, authorityProvider);
+ }
+
+ /**
+ * Tests the general case when a user account is created.
+ *
+ * @throws Exception ex
+ */
+ @Test
+ public void testAccountCreation() throws Exception {
+ AuthorizeUserAction authorizeUser = new AuthorizeUserAction(USER_DN_6, 0);
+ NiFiUser user = authorizeUser.execute(daoFactory, authorityProvider);
+
+ // verify the user
+ Assert.assertEquals(USER_DN_6, user.getDn());
+ Assert.assertEquals(1, user.getAuthorities().size());
+ Assert.assertTrue(user.getAuthorities().contains(Authority.ROLE_MONITOR));
+
+ // verify interaction with dao and provider
+ Mockito.verify(userDao, Mockito.times(1)).createUser(user);
+ }
+
+ /**
+ * Tests the general case when there is an existing user account that
+ * requires verification.
+ *
+ * @throws Exception ex
+ */
+ @Test
+ public void testExistingUserRequiresVerification() throws Exception {
+ AuthorizeUserAction authorizeUser = new AuthorizeUserAction(USER_DN_7, 0);
+ NiFiUser user = authorizeUser.execute(daoFactory, authorityProvider);
+
+ // verify the user
+ Assert.assertEquals(USER_DN_7, user.getDn());
+ Assert.assertEquals(1, user.getAuthorities().size());
+ Assert.assertTrue(user.getAuthorities().contains(Authority.ROLE_DFM));
+
+ // verify interaction with dao and provider
+ Mockito.verify(userDao, Mockito.times(1)).updateUser(user);
+ Mockito.verify(authorityDao, Mockito.times(1)).createAuthorities(EnumSet.of(Authority.ROLE_DFM), USER_ID_7);
+ }
+
+ /**
+ * Tests the general case when there is an existing user account that does
+ * not require verification.
+ *
+ * @throws Exception ex
+ */
+ @Test
+ public void testExistingUserNoVerification() throws Exception {
+ // disabling verification by passing in a large cache duration
+ AuthorizeUserAction authorizeUser = new AuthorizeUserAction(USER_DN_8, Integer.MAX_VALUE);
+ NiFiUser user = authorizeUser.execute(daoFactory, authorityProvider);
+
+ // verify the user
+ Assert.assertEquals(USER_DN_8, user.getDn());
+ Assert.assertEquals(1, user.getAuthorities().size());
+ Assert.assertTrue(user.getAuthorities().contains(Authority.ROLE_MONITOR));
+
+ // verify interaction with dao and provider
+ Mockito.verify(userDao, Mockito.times(1)).updateUser(user);
+ Mockito.verify(authorityDao, Mockito.never()).createAuthorities(Mockito.anySet(), Mockito.eq(USER_ID_8));
+ Mockito.verify(authorityDao, Mockito.never()).deleteAuthorities(Mockito.anySet(), Mockito.eq(USER_ID_8));
+ }
+
+ /**
+ * Tests existing users whose accounts are in a pending status.
+ *
+ * @throws Exception ex
+ */
+ @Test(expected = AccountPendingException.class)
+ public void testExistingPendingUser() throws Exception {
+ // disabling verification by passing in a large cache duration
+ AuthorizeUserAction authorizeUser = new AuthorizeUserAction(USER_DN_9, Integer.MAX_VALUE);
+ authorizeUser.execute(daoFactory, authorityProvider);
+ }
+
+ /**
+ * Tests existing users whose accounts are in a disabled status.
+ *
+ * @throws Exception ex
+ */
+ @Test(expected = AccountDisabledException.class)
+ public void testExistingDisabledUser() throws Exception {
+ // disabling verification by passing in a large cache duration
+ AuthorizeUserAction authorizeUser = new AuthorizeUserAction(USER_DN_10, Integer.MAX_VALUE);
+ authorizeUser.execute(daoFactory, authorityProvider);
+ }
+
+ /**
+ * Tests the general case where there is an active user that has been
+ * removed from the authority provider.
+ *
+ * @throws Exception ex
+ */
+ @Test
+ public void testExistingActiveUserNotFoundInProvider() throws Exception {
+ try {
+ AuthorizeUserAction authorizeUser = new AuthorizeUserAction(USER_DN_11, 0);
+ authorizeUser.execute(daoFactory, authorityProvider);
+
+ Assert.fail();
+ } catch (AccountDisabledException ade) {
+ ArgumentCaptor<NiFiUser> user = ArgumentCaptor.forClass(NiFiUser.class);
+
+ // verify interaction with dao
+ Mockito.verify(userDao, Mockito.times(1)).updateUser(user.capture());
+
+ // verify user
+ Assert.assertEquals(AccountStatus.DISABLED, user.getValue().getStatus());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/CreateUserActionTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/CreateUserActionTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/CreateUserActionTest.java
new file mode 100644
index 0000000..6486d32
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/CreateUserActionTest.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import java.util.EnumSet;
+import java.util.Set;
+import org.apache.nifi.admin.dao.AuthorityDAO;
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.admin.dao.DataAccessException;
+import org.apache.nifi.admin.dao.UserDAO;
+import org.apache.nifi.authorization.Authority;
+import org.apache.nifi.user.NiFiUser;
+import org.apache.commons.lang3.StringUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+/**
+ * Test cases for creating a user.
+ */
+public class CreateUserActionTest {
+
+ private final String USER_ID_2 = "2";
+ private final String USER_ID_3 = "3";
+
+ private final String USER_DN_1 = "data access exception when creating user";
+ private final String USER_DN_3 = "general create user case";
+
+ private DAOFactory daoFactory;
+ private UserDAO userDao;
+ private AuthorityDAO authorityDao;
+
+ @Before
+ public void setup() throws Exception {
+ // mock the user dao
+ userDao = Mockito.mock(UserDAO.class);
+ Mockito.doAnswer(new Answer<Void>() {
+ @Override
+ public Void answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ NiFiUser user = (NiFiUser) args[0];
+
+ if (USER_DN_1.equals(user.getDn())) {
+ throw new DataAccessException();
+ } else if (USER_DN_3.equals(user.getDn())) {
+ user.setId(USER_ID_3);
+ }
+
+ // do nothing
+ return null;
+ }
+ }).when(userDao).createUser(Mockito.any(NiFiUser.class));
+
+ // mock the authority dao
+ authorityDao = Mockito.mock(AuthorityDAO.class);
+ Mockito.doAnswer(new Answer<Void>() {
+ @Override
+ public Void answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ Set<Authority> authorities = (Set<Authority>) args[0];
+ String id = (String) args[1];
+
+ if (USER_ID_2.equals(id)) {
+ throw new DataAccessException(StringUtils.EMPTY);
+ }
+
+ // do nothing
+ return null;
+ }
+ }).when(authorityDao).createAuthorities(Mockito.anySetOf(Authority.class), Mockito.anyString());
+
+ // mock the dao factory
+ daoFactory = Mockito.mock(DAOFactory.class);
+ Mockito.when(daoFactory.getUserDAO()).thenReturn(userDao);
+ Mockito.when(daoFactory.getAuthorityDAO()).thenReturn(authorityDao);
+ }
+
+ /**
+ * Tests DataAccessExceptions that occur while creating user accounts.
+ *
+ * @throws Exception ex
+ */
+ @Test(expected = DataAccessException.class)
+ public void testExceptionCreatingUser() throws Exception {
+ NiFiUser user = new NiFiUser();
+ user.setDn(USER_DN_1);
+
+ CreateUserAction createUser = new CreateUserAction(user);
+ createUser.execute(daoFactory, null);
+ }
+
+ /**
+ * Tests DataAccessExceptions that occur while create user authorities.
+ *
+ * @throws Exception ex
+ */
+ @Test(expected = DataAccessException.class)
+ public void testExceptionCreatingAuthoroties() throws Exception {
+ NiFiUser user = new NiFiUser();
+ user.setId(USER_ID_2);
+
+ CreateUserAction createUser = new CreateUserAction(user);
+ createUser.execute(daoFactory, null);
+ }
+
+ /**
+ * General case for creating a user.
+ *
+ * @throws Exception ex
+ */
+ @Test
+ public void testCreateUserAccount() throws Exception {
+ NiFiUser user = new NiFiUser();
+ user.setDn(USER_DN_3);
+ user.getAuthorities().addAll(EnumSet.of(Authority.ROLE_DFM, Authority.ROLE_ADMIN));
+
+ CreateUserAction createUser = new CreateUserAction(user);
+ createUser.execute(daoFactory, null);
+
+ // verify the user
+ Assert.assertEquals(USER_ID_3, user.getId());
+
+ // verify interaction with dao
+ Mockito.verify(userDao, Mockito.times(1)).createUser(user);
+ Mockito.verify(authorityDao, Mockito.times(1)).createAuthorities(user.getAuthorities(), USER_ID_3);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/DisableUserActionTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/DisableUserActionTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/DisableUserActionTest.java
new file mode 100644
index 0000000..b0e1ac1
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/DisableUserActionTest.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.admin.dao.DataAccessException;
+import org.apache.nifi.admin.dao.UserDAO;
+import org.apache.nifi.admin.service.AccountNotFoundException;
+import org.apache.nifi.admin.service.AdministrationException;
+import org.apache.nifi.authorization.AuthorityProvider;
+import org.apache.nifi.authorization.exception.AuthorityAccessException;
+import org.apache.nifi.user.AccountStatus;
+import org.apache.nifi.user.NiFiUser;
+import org.apache.commons.lang3.StringUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+public class DisableUserActionTest {
+
+ private static final String USER_ID_1 = "1";
+ private static final String USER_ID_2 = "2";
+ private static final String USER_ID_3 = "3";
+ private static final String USER_ID_4 = "4";
+
+ private static final String USER_DN_3 = "authority access exception";
+ private static final String USER_DN_4 = "general disable user case";
+
+ private DAOFactory daoFactory;
+ private UserDAO userDao;
+ private AuthorityProvider authorityProvider;
+
+ @Before
+ public void setup() throws Exception {
+ // mock the user dao
+ userDao = Mockito.mock(UserDAO.class);
+ Mockito.doAnswer(new Answer<NiFiUser>() {
+ @Override
+ public NiFiUser answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ String id = (String) args[0];
+
+ NiFiUser user = null;
+ if (USER_ID_1.equals(id)) {
+ // leave user uninitialized
+ } else if (USER_ID_2.equals(id)) {
+ user = new NiFiUser();
+ user.setId(id);
+ } else if (USER_ID_3.equals(id)) {
+ user = new NiFiUser();
+ user.setId(id);
+ user.setDn(USER_DN_3);
+ } else if (USER_ID_4.equals(id)) {
+ user = new NiFiUser();
+ user.setId(id);
+ user.setDn(USER_DN_4);
+ user.setStatus(AccountStatus.ACTIVE);
+ }
+ return user;
+ }
+ }).when(userDao).findUserById(Mockito.anyString());
+ Mockito.doAnswer(new Answer<Void>() {
+ @Override
+ public Void answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ NiFiUser user = (NiFiUser) args[0];
+
+ if (USER_ID_2.equals(user.getId())) {
+ throw new DataAccessException(StringUtils.EMPTY);
+ }
+
+ // do nothing
+ return null;
+ }
+ }).when(userDao).updateUser(Mockito.any(NiFiUser.class));
+
+ // mock the dao factory
+ daoFactory = Mockito.mock(DAOFactory.class);
+ Mockito.when(daoFactory.getUserDAO()).thenReturn(userDao);
+
+ // mock the authority provider
+ authorityProvider = Mockito.mock(AuthorityProvider.class);
+ Mockito.doAnswer(new Answer<Void>() {
+ @Override
+ public Void answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ String dn = (String) args[0];
+
+ if (USER_DN_3.equals(dn)) {
+ throw new AuthorityAccessException(StringUtils.EMPTY);
+ }
+
+ // do nothing
+ return null;
+ }
+ }).when(authorityProvider).revokeUser(Mockito.anyString());
+ }
+
+ /**
+ * Tests the case when the user account is unknown.
+ *
+ * @throws Exception ex
+ */
+ @Test(expected = AccountNotFoundException.class)
+ public void testUnknownUserAccount() throws Exception {
+ DisableUserAction disableUser = new DisableUserAction(USER_ID_1);
+ disableUser.execute(daoFactory, authorityProvider);
+ }
+
+ /**
+ * Tests the case when a DataAccessException is thrown by the userDao.
+ *
+ * @throws Exception ex
+ */
+ @Test(expected = DataAccessException.class)
+ public void testDataAccessExceptionInUserDao() throws Exception {
+ DisableUserAction disableUser = new DisableUserAction(USER_ID_2);
+ disableUser.execute(daoFactory, authorityProvider);
+ }
+
+ /**
+ * Tests the case when a AuthorityAccessException is thrown by the provider.
+ *
+ * @throws Exception ex
+ */
+ @Test(expected = AdministrationException.class)
+ public void testAuthorityAccessExceptionInProvider() throws Exception {
+ DisableUserAction disableUser = new DisableUserAction(USER_ID_3);
+ disableUser.execute(daoFactory, authorityProvider);
+ }
+
+ /**
+ * Tests the general case when the user is disabled.
+ *
+ * @throws Exception ex
+ */
+ @Test
+ public void testDisableUser() throws Exception {
+ DisableUserAction disableUser = new DisableUserAction(USER_ID_4);
+ NiFiUser user = disableUser.execute(daoFactory, authorityProvider);
+
+ // verify the user
+ Assert.assertEquals(USER_ID_4, user.getId());
+ Assert.assertEquals(USER_DN_4, user.getDn());
+ Assert.assertEquals(AccountStatus.DISABLED, user.getStatus());
+
+ // verify the interaction with the dao and provider
+ Mockito.verify(userDao, Mockito.times(1)).updateUser(user);
+ Mockito.verify(authorityProvider, Mockito.times(1)).revokeUser(USER_DN_4);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/InvalidateUserAccountActionTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/InvalidateUserAccountActionTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/InvalidateUserAccountActionTest.java
new file mode 100644
index 0000000..cffd280
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/InvalidateUserAccountActionTest.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import java.util.Date;
+import org.junit.Assert;
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.admin.dao.DataAccessException;
+import org.apache.nifi.admin.dao.UserDAO;
+import org.apache.nifi.admin.service.AccountNotFoundException;
+import org.apache.nifi.user.NiFiUser;
+import org.apache.commons.lang3.StringUtils;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+/**
+ * Test case for InvalidateUserAccountAction.
+ */
+public class InvalidateUserAccountActionTest {
+
+ private static final String USER_ID_1 = "1";
+ private static final String USER_ID_2 = "2";
+ private static final String USER_ID_3 = "3";
+
+ private DAOFactory daoFactory;
+ private UserDAO userDao;
+
+ @Before
+ public void setup() throws Exception {
+ // mock the user dao
+ userDao = Mockito.mock(UserDAO.class);
+ Mockito.doAnswer(new Answer<NiFiUser>() {
+ @Override
+ public NiFiUser answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ String id = (String) args[0];
+
+ NiFiUser user = null;
+ if (USER_ID_1.equals(id)) {
+ // leave uninitialized
+ } else if (USER_ID_2.equals(id)) {
+ user = new NiFiUser();
+ user.setId(USER_ID_2);
+ } else if (USER_ID_3.equals(id)) {
+ user = new NiFiUser();
+ user.setId(USER_ID_3);
+ user.setLastVerified(new Date());
+ }
+ return user;
+ }
+ }).when(userDao).findUserById(Mockito.anyString());
+ Mockito.doAnswer(new Answer<Void>() {
+ @Override
+ public Void answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ NiFiUser user = (NiFiUser) args[0];
+
+ if (USER_ID_2.equals(user.getId())) {
+ throw new DataAccessException(StringUtils.EMPTY);
+ }
+
+ // do nothing
+ return null;
+ }
+ }).when(userDao).updateUser(Mockito.any(NiFiUser.class));
+
+ // mock the dao factory
+ daoFactory = Mockito.mock(DAOFactory.class);
+ Mockito.when(daoFactory.getUserDAO()).thenReturn(userDao);
+ }
+
+ @Test(expected = AccountNotFoundException.class)
+ public void testAccountNotFoundException() throws Exception {
+ InvalidateUserAccountAction invalidateUserAccount = new InvalidateUserAccountAction(USER_ID_1);
+ invalidateUserAccount.execute(daoFactory, null);
+ }
+
+ /**
+ * Tests when a data access exception occurs when updating the user record.
+ *
+ * @throws Exception ex
+ */
+ @Test(expected = DataAccessException.class)
+ public void testDataAccessException() throws Exception {
+ InvalidateUserAccountAction invalidateUserAccount = new InvalidateUserAccountAction(USER_ID_2);
+ invalidateUserAccount.execute(daoFactory, null);
+ }
+
+ /**
+ * Tests the general case of invalidating a user.
+ *
+ * @throws Exception ex
+ */
+ @Test
+ public void testInvalidateUser() throws Exception {
+ InvalidateUserAccountAction invalidateUserAccount = new InvalidateUserAccountAction(USER_ID_3);
+ invalidateUserAccount.execute(daoFactory, null);
+
+ // verify the interaction with the dao
+ ArgumentCaptor<NiFiUser> userCaptor = ArgumentCaptor.forClass(NiFiUser.class);
+ Mockito.verify(userDao, Mockito.times(1)).updateUser(userCaptor.capture());
+
+ // verify the user
+ NiFiUser user = userCaptor.getValue();
+ Assert.assertEquals(USER_ID_3, user.getId());
+ Assert.assertNull(user.getLastVerified());
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/RequestUserAccountActionTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/RequestUserAccountActionTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/RequestUserAccountActionTest.java
new file mode 100644
index 0000000..7707b2c
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/RequestUserAccountActionTest.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.admin.dao.DataAccessException;
+import org.apache.nifi.admin.dao.UserDAO;
+import org.apache.nifi.user.AccountStatus;
+import org.apache.nifi.user.NiFiUser;
+import org.apache.commons.lang3.StringUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+/**
+ * Test case for RequestUserAccountAction.
+ */
+public class RequestUserAccountActionTest {
+
+ private static final String USER_ID_3 = "3";
+
+ private static final String USER_DN_1 = "existing user account dn";
+ private static final String USER_DN_2 = "data access exception";
+ private static final String USER_DN_3 = "new account request";
+
+ private DAOFactory daoFactory;
+ private UserDAO userDao;
+
+ @Before
+ public void setup() throws Exception {
+ // mock the user dao
+ userDao = Mockito.mock(UserDAO.class);
+ Mockito.doAnswer(new Answer<NiFiUser>() {
+ @Override
+ public NiFiUser answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ String dn = (String) args[0];
+
+ NiFiUser user = null;
+ if (USER_DN_1.equals(dn)) {
+ user = new NiFiUser();
+ }
+ return user;
+ }
+ }).when(userDao).findUserByDn(Mockito.anyString());
+ Mockito.doAnswer(new Answer<Void>() {
+ @Override
+ public Void answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ NiFiUser user = (NiFiUser) args[0];
+ switch (user.getDn()) {
+ case USER_DN_2:
+ throw new DataAccessException();
+ case USER_DN_3:
+ user.setId(USER_ID_3);
+ break;
+ }
+
+ // do nothing
+ return null;
+ }
+ }).when(userDao).createUser(Mockito.any(NiFiUser.class));
+
+ // mock the dao factory
+ daoFactory = Mockito.mock(DAOFactory.class);
+ Mockito.when(daoFactory.getUserDAO()).thenReturn(userDao);
+ }
+
+ /**
+ * Tests when a user account already exists.
+ *
+ * @throws Exception ex
+ */
+ @Test(expected = IllegalArgumentException.class)
+ public void testExistingAccount() throws Exception {
+ RequestUserAccountAction requestUserAccount = new RequestUserAccountAction(USER_DN_1, StringUtils.EMPTY);
+ requestUserAccount.execute(daoFactory, null);
+ }
+
+ /**
+ * Tests when a DataAccessException occurs while saving the new account
+ * request.
+ *
+ * @throws Exception ex
+ */
+ @Test(expected = DataAccessException.class)
+ public void testDataAccessException() throws Exception {
+ RequestUserAccountAction requestUserAccount = new RequestUserAccountAction(USER_DN_2, StringUtils.EMPTY);
+ requestUserAccount.execute(daoFactory, null);
+ }
+
+ /**
+ * Tests the general case for requesting a new user account.
+ *
+ * @throws Exception ex
+ */
+ @Test
+ public void testRequestUserAccountAction() throws Exception {
+ RequestUserAccountAction requestUserAccount = new RequestUserAccountAction(USER_DN_3, StringUtils.EMPTY);
+ NiFiUser user = requestUserAccount.execute(daoFactory, null);
+
+ // verfiy the user
+ Assert.assertEquals(USER_ID_3, user.getId());
+ Assert.assertEquals(USER_DN_3, user.getDn());
+ Assert.assertEquals(AccountStatus.PENDING, user.getStatus());
+
+ // verify interaction with dao
+ Mockito.verify(userDao, Mockito.times(1)).createUser(user);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/SeedUserAccountsActionTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/SeedUserAccountsActionTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/SeedUserAccountsActionTest.java
new file mode 100644
index 0000000..652d992
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/SeedUserAccountsActionTest.java
@@ -0,0 +1,262 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.Set;
+import org.apache.nifi.admin.dao.AuthorityDAO;
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.admin.dao.UserDAO;
+import org.apache.nifi.authorization.Authority;
+import org.apache.nifi.authorization.AuthorityProvider;
+import org.apache.nifi.user.AccountStatus;
+import org.apache.nifi.user.NiFiUser;
+import org.hamcrest.Matcher;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.ArgumentMatcher;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+/**
+ *
+ */
+public class SeedUserAccountsActionTest {
+
+ private static final String USER_ID_1 = "1";
+ private static final String USER_ID_2 = "2";
+ private static final String USER_ID_3 = "3";
+ private static final String USER_ID_4 = "4";
+
+ private static final String USER_DN_1 = "user dn 1 - active user - remove monitor and operator, add dfm";
+ private static final String USER_DN_2 = "user dn 2 - active user - no action";
+ private static final String USER_DN_3 = "user dn 3 - pending user - add operator";
+ private static final String USER_DN_4 = "user dn 4 - new user - add monitor";
+
+ private DAOFactory daoFactory;
+ private UserDAO userDao;
+ private AuthorityDAO authorityDao;
+ private AuthorityProvider authorityProvider;
+
+ @Before
+ public void setup() throws Exception {
+ // mock the user dao
+ userDao = Mockito.mock(UserDAO.class);
+ Mockito.doAnswer(new Answer<NiFiUser>() {
+ @Override
+ public NiFiUser answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ String id = (String) args[0];
+
+ NiFiUser user = null;
+ if (USER_ID_1.equals(id)) {
+ user = new NiFiUser();
+ user.setId(USER_ID_1);
+ user.setDn(USER_DN_1);
+ user.getAuthorities().addAll(EnumSet.of(Authority.ROLE_MONITOR));
+ user.setStatus(AccountStatus.ACTIVE);
+ } else if (USER_ID_2.equals(id)) {
+ user = new NiFiUser();
+ user.setId(USER_ID_2);
+ user.setDn(USER_DN_2);
+ user.getAuthorities().addAll(EnumSet.of(Authority.ROLE_ADMIN));
+ user.setStatus(AccountStatus.ACTIVE);
+ } else if (USER_ID_3.equals(id)) {
+ user = new NiFiUser();
+ user.setId(USER_ID_3);
+ user.setDn(USER_DN_3);
+ user.setStatus(AccountStatus.PENDING);
+ }
+ return user;
+ }
+ }).when(userDao).findUserById(Mockito.anyString());
+ Mockito.doAnswer(new Answer<NiFiUser>() {
+ @Override
+ public NiFiUser answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ String dn = (String) args[0];
+
+ NiFiUser user = null;
+ if (USER_DN_1.equals(dn)) {
+ user = new NiFiUser();
+ user.setId(USER_ID_1);
+ user.setDn(USER_DN_1);
+ user.getAuthorities().addAll(EnumSet.of(Authority.ROLE_MONITOR));
+ user.setStatus(AccountStatus.ACTIVE);
+ } else if (USER_DN_2.equals(dn)) {
+ user = new NiFiUser();
+ user.setId(USER_ID_2);
+ user.setDn(USER_DN_2);
+ user.getAuthorities().addAll(EnumSet.of(Authority.ROLE_ADMIN));
+ user.setStatus(AccountStatus.ACTIVE);
+ } else if (USER_DN_3.equals(dn)) {
+ user = new NiFiUser();
+ user.setId(USER_ID_3);
+ user.setDn(USER_DN_3);
+ user.setStatus(AccountStatus.PENDING);
+ }
+ return user;
+ }
+ }).when(userDao).findUserByDn(Mockito.anyString());
+ Mockito.doAnswer(new Answer<Void>() {
+ @Override
+ public Void answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ NiFiUser user = (NiFiUser) args[0];
+
+ if (USER_DN_4.equals(user.getDn())) {
+ user.setId(USER_ID_4);
+ }
+
+ return null;
+ }
+ }).when(userDao).createUser(Mockito.any(NiFiUser.class));
+
+ // mock the authority dao
+ authorityDao = Mockito.mock(AuthorityDAO.class);
+
+ // mock the authority provider
+ authorityProvider = Mockito.mock(AuthorityProvider.class);
+ Mockito.doAnswer(new Answer<Set<String>>() {
+ @Override
+ public Set<String> answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ Authority role = (Authority) args[0];
+
+ Set<String> users = new HashSet<>();
+ if (Authority.ROLE_DFM.equals(role)) {
+ users.add(USER_DN_1);
+ } else if (Authority.ROLE_ADMIN.equals(role)) {
+ users.add(USER_DN_2);
+ } else if (Authority.ROLE_PROXY.equals(role)) {
+ users.add(USER_DN_3);
+ } else if (Authority.ROLE_MONITOR.equals(role)) {
+ users.add(USER_DN_4);
+ }
+ return users;
+ }
+ }).when(authorityProvider).getUsers(Mockito.any(Authority.class));
+ Mockito.doAnswer(new Answer<Set<Authority>>() {
+ @Override
+ public Set<Authority> answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ String dn = (String) args[0];
+
+ Set<Authority> authorities = EnumSet.noneOf(Authority.class);
+ switch (dn) {
+ case USER_DN_1:
+ authorities.add(Authority.ROLE_DFM);
+ break;
+ case USER_DN_2:
+ authorities.add(Authority.ROLE_ADMIN);
+ break;
+ case USER_DN_3:
+ authorities.add(Authority.ROLE_PROXY);
+ break;
+ case USER_DN_4:
+ authorities.add(Authority.ROLE_MONITOR);
+ break;
+ }
+ return authorities;
+ }
+ }).when(authorityProvider).getAuthorities(Mockito.anyString());
+
+ // mock the dao factory
+ daoFactory = Mockito.mock(DAOFactory.class);
+ Mockito.when(daoFactory.getUserDAO()).thenReturn(userDao);
+ Mockito.when(daoFactory.getAuthorityDAO()).thenReturn(authorityDao);
+ }
+
+ /**
+ * Tests seeding the user accounts.
+ *
+ * @throws Exception ex
+ */
+ @Test
+ public void testSeedUsers() throws Exception {
+ SeedUserAccountsAction seedUserAccounts = new SeedUserAccountsAction();
+ seedUserAccounts.execute(daoFactory, authorityProvider);
+
+ // matcher for user 1
+ Matcher<NiFiUser> matchesUser1 = new ArgumentMatcher<NiFiUser>() {
+ @Override
+ public boolean matches(Object argument) {
+ NiFiUser user = (NiFiUser) argument;
+ return USER_ID_1.equals(user.getId());
+ }
+ };
+
+ // verify user 1 - active existing user - remove monitor, operator, add dfm
+ Mockito.verify(userDao, Mockito.times(1)).updateUser(Mockito.argThat(matchesUser1));
+ Mockito.verify(userDao, Mockito.never()).createUser(Mockito.argThat(matchesUser1));
+ Mockito.verify(authorityDao, Mockito.times(1)).createAuthorities(EnumSet.of(Authority.ROLE_DFM), USER_ID_1);
+
+ // matcher for user 2
+ Matcher<NiFiUser> matchesUser2 = new ArgumentMatcher<NiFiUser>() {
+ @Override
+ public boolean matches(Object argument) {
+ NiFiUser user = (NiFiUser) argument;
+ return USER_ID_2.equals(user.getId());
+ }
+ };
+
+ // verify user 2 - active existing user - no actions
+ Mockito.verify(userDao, Mockito.times(1)).updateUser(Mockito.argThat(matchesUser2));
+ Mockito.verify(userDao, Mockito.never()).createUser(Mockito.argThat(matchesUser2));
+ Mockito.verify(authorityDao, Mockito.never()).createAuthorities(Mockito.anySet(), Mockito.eq(USER_ID_2));
+ Mockito.verify(authorityDao, Mockito.never()).deleteAuthorities(Mockito.anySet(), Mockito.eq(USER_ID_2));
+
+ // matchers for user 3
+ Matcher<NiFiUser> matchesPendingUser3 = new ArgumentMatcher<NiFiUser>() {
+ @Override
+ public boolean matches(Object argument) {
+ NiFiUser user = (NiFiUser) argument;
+ return USER_ID_3.equals(user.getId()) && AccountStatus.ACTIVE.equals(user.getStatus());
+ }
+ };
+ Matcher<NiFiUser> matchesUser3 = new ArgumentMatcher<NiFiUser>() {
+ @Override
+ public boolean matches(Object argument) {
+ NiFiUser user = (NiFiUser) argument;
+ return USER_ID_3.equals(user.getId());
+ }
+ };
+
+ // verify user 3 - pending user - add operator
+ Mockito.verify(userDao, Mockito.times(1)).updateUser(Mockito.argThat(matchesPendingUser3));
+ Mockito.verify(userDao, Mockito.never()).createUser(Mockito.argThat(matchesUser3));
+ Mockito.verify(authorityDao, Mockito.times(1)).createAuthorities(EnumSet.of(Authority.ROLE_PROXY), USER_ID_3);
+ Mockito.verify(authorityDao, Mockito.never()).deleteAuthorities(Mockito.anySet(), Mockito.eq(USER_ID_3));
+
+ // matcher for user 4
+ Matcher<NiFiUser> matchesUser4 = new ArgumentMatcher<NiFiUser>() {
+ @Override
+ public boolean matches(Object argument) {
+ NiFiUser user = (NiFiUser) argument;
+ return USER_ID_4.equals(user.getId());
+ }
+ };
+
+ // verify user 4 - new user - add monitor
+ Mockito.verify(userDao, Mockito.never()).updateUser(Mockito.argThat(matchesUser4));
+ Mockito.verify(userDao, Mockito.times(1)).createUser(Mockito.argThat(matchesUser4));
+ Mockito.verify(authorityDao, Mockito.times(1)).createAuthorities(EnumSet.of(Authority.ROLE_MONITOR), USER_ID_4);
+ Mockito.verify(authorityDao, Mockito.never()).deleteAuthorities(Mockito.anySet(), Mockito.eq(USER_ID_4));
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/SetUserAuthoritiesActionTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/SetUserAuthoritiesActionTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/SetUserAuthoritiesActionTest.java
new file mode 100644
index 0000000..22504f7
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/SetUserAuthoritiesActionTest.java
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.Set;
+import org.apache.nifi.admin.dao.AuthorityDAO;
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.admin.dao.UserDAO;
+import org.apache.nifi.admin.service.AccountNotFoundException;
+import org.apache.nifi.admin.service.AdministrationException;
+import org.apache.nifi.authorization.Authority;
+import org.apache.nifi.authorization.AuthorityProvider;
+import org.apache.nifi.authorization.exception.AuthorityAccessException;
+import org.apache.nifi.user.AccountStatus;
+import org.apache.nifi.user.NiFiUser;
+import org.apache.commons.lang3.StringUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+/**
+ * Test case for SetUserAuthoritiesAction.
+ */
+public class SetUserAuthoritiesActionTest {
+
+ private static final String USER_ID_1 = "1";
+ private static final String USER_ID_2 = "2";
+ private static final String USER_ID_3 = "3";
+
+ private static final String USER_DN_2 = "user dn 2";
+ private static final String USER_DN_3 = "user dn 3";
+
+ private DAOFactory daoFactory;
+ private UserDAO userDao;
+ private AuthorityDAO authorityDao;
+ private AuthorityProvider authorityProvider;
+
+ @Before
+ public void setup() throws Exception {
+ // mock the user dao
+ userDao = Mockito.mock(UserDAO.class);
+ Mockito.doAnswer(new Answer<NiFiUser>() {
+ @Override
+ public NiFiUser answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ String id = (String) args[0];
+
+ NiFiUser user = null;
+ if (USER_ID_1.equals(id)) {
+ // leave user uninitialized
+ } else if (USER_ID_2.equals(id)) {
+ user = new NiFiUser();
+ user.setId(USER_ID_2);
+ user.setDn(USER_DN_2);
+ } else if (USER_ID_3.equals(id)) {
+ user = new NiFiUser();
+ user.setId(USER_ID_3);
+ user.setDn(USER_DN_3);
+ user.getAuthorities().addAll(EnumSet.of(Authority.ROLE_MONITOR));
+ user.setStatus(AccountStatus.ACTIVE);
+ }
+ return user;
+ }
+ }).when(userDao).findUserById(Mockito.anyString());
+ Mockito.doAnswer(new Answer<NiFiUser>() {
+ @Override
+ public NiFiUser answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ String dn = (String) args[0];
+
+ NiFiUser user = null;
+ if (USER_DN_3.equals(dn)) {
+ user = new NiFiUser();
+ user.setId(USER_ID_3);
+ user.setDn(USER_DN_3);
+ user.getAuthorities().addAll(EnumSet.of(Authority.ROLE_MONITOR));
+ user.setStatus(AccountStatus.ACTIVE);
+ }
+ return user;
+ }
+ }).when(userDao).findUserByDn(Mockito.anyString());
+ Mockito.doAnswer(new Answer<Void>() {
+ @Override
+ public Void answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ NiFiUser user = (NiFiUser) args[0];
+
+ // do nothing
+ return null;
+ }
+ }).when(userDao).updateUser(Mockito.any(NiFiUser.class));
+
+ // mock the authority dao
+ authorityDao = Mockito.mock(AuthorityDAO.class);
+ Mockito.doAnswer(new Answer<Void>() {
+ @Override
+ public Void answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ Set<Authority> authorities = (Set<Authority>) args[0];
+ String id = (String) args[1];
+
+ // do nothing
+ return null;
+ }
+ }).when(authorityDao).createAuthorities(Mockito.anySetOf(Authority.class), Mockito.anyString());
+ Mockito.doAnswer(new Answer<Void>() {
+ @Override
+ public Void answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ Set<Authority> authorities = (Set<Authority>) args[0];
+ String id = (String) args[1];
+
+ // do nothing
+ return null;
+ }
+ }).when(authorityDao).deleteAuthorities(Mockito.anySetOf(Authority.class), Mockito.anyString());
+
+ // mock the dao factory
+ daoFactory = Mockito.mock(DAOFactory.class);
+ Mockito.when(daoFactory.getUserDAO()).thenReturn(userDao);
+ Mockito.when(daoFactory.getAuthorityDAO()).thenReturn(authorityDao);
+
+ // mock the authority provider
+ authorityProvider = Mockito.mock(AuthorityProvider.class);
+ Mockito.doAnswer(new Answer<Set<Authority>>() {
+ @Override
+ public Set<Authority> answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ String dn = (String) args[0];
+
+ Set<Authority> authorities = EnumSet.noneOf(Authority.class);
+ if (USER_DN_3.equals(dn)) {
+ authorities.add(Authority.ROLE_DFM);
+ }
+
+ return authorities;
+ }
+ }).when(authorityProvider).getAuthorities(Mockito.anyString());
+ Mockito.doAnswer(new Answer<Void>() {
+ @Override
+ public Void answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ String dn = (String) args[0];
+ Set<Authority> authorites = (Set<Authority>) args[1];
+
+ if (USER_DN_2.equals(dn)) {
+ throw new AuthorityAccessException(StringUtils.EMPTY);
+ }
+
+ // do nothing
+ return null;
+ }
+ }).when(authorityProvider).setAuthorities(Mockito.anyString(), Mockito.anySet());
+ }
+
+ /**
+ * Test activating an unknown user account. User accounts are unknown then
+ * there is no pending account for the user.
+ *
+ * @throws Exception ex
+ */
+ @Test(expected = AccountNotFoundException.class)
+ public void testUnknownUser() throws Exception {
+ UpdateUserAction setUserAuthorities = new UpdateUserAction(USER_ID_1, Collections.EMPTY_SET);
+ setUserAuthorities.execute(daoFactory, authorityProvider);
+ }
+
+ /**
+ * Testing case then an AuthorityAccessException occurs while setting a
+ * users authorities.
+ *
+ * @throws Exception ex
+ */
+ @Test(expected = AdministrationException.class)
+ public void testAuthorityAccessException() throws Exception {
+ UpdateUserAction setUserAuthorities = new UpdateUserAction(USER_ID_2, Collections.EMPTY_SET);
+ setUserAuthorities.execute(daoFactory, authorityProvider);
+ }
+
+ /**
+ * Tests general case of setting user authorities.
+ *
+ * @throws Exception ex
+ */
+ @Test
+ public void testSetAuthorities() throws Exception {
+ UpdateUserAction setUserAuthorities = new UpdateUserAction(USER_ID_3, EnumSet.of(Authority.ROLE_ADMIN));
+ NiFiUser user = setUserAuthorities.execute(daoFactory, authorityProvider);
+
+ // verify user
+ Assert.assertEquals(USER_ID_3, user.getId());
+ Assert.assertEquals(1, user.getAuthorities().size());
+ Assert.assertTrue(user.getAuthorities().contains(Authority.ROLE_ADMIN));
+
+ // verify interaction with dao
+ Mockito.verify(userDao, Mockito.times(1)).updateUser(user);
+ Mockito.verify(authorityDao, Mockito.times(1)).createAuthorities(EnumSet.of(Authority.ROLE_ADMIN), USER_ID_3);
+
+ Set<Authority> authoritiesAddedToProvider = EnumSet.of(Authority.ROLE_ADMIN);
+
+ // verify interaction with provider
+ Mockito.verify(authorityProvider, Mockito.times(1)).setAuthorities(USER_DN_3, authoritiesAddedToProvider);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/.gitignore
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/.gitignore b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/.gitignore
new file mode 100755
index 0000000..cd1a4e7
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/.gitignore
@@ -0,0 +1,6 @@
+/target
+/target
+/target
+/target
+/target
+/target
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/pom.xml
new file mode 100644
index 0000000..bf4177f
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/pom.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-framework</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>nifi-client-dto</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>com.wordnik</groupId>
+ <artifactId>swagger-annotations</artifactId>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/AboutDTO.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/AboutDTO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/AboutDTO.java
new file mode 100644
index 0000000..83aad41
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/AboutDTO.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.api.dto;
+
+import com.wordnik.swagger.annotations.ApiModelProperty;
+import javax.xml.bind.annotation.XmlType;
+
+/**
+ * Contains details about this NiFi including the title and version.
+ */
+@XmlType(name = "about")
+public class AboutDTO {
+
+ private String title;
+ private String version;
+
+ /* getters / setters */
+ /**
+ * The title to be used on the page and in the About dialog.
+ *
+ * @return The title
+ */
+ @ApiModelProperty(
+ value = "The title to be used on the page and in the about dialog."
+ )
+ public String getTitle() {
+ return title;
+ }
+
+ public void setTitle(String title) {
+ this.title = title;
+ }
+
+ /**
+ * The version of this NiFi.
+ *
+ * @return The version.
+ */
+ @ApiModelProperty(
+ value = "The version of this NiFi."
+ )
+ public String getVersion() {
+ return version;
+ }
+
+ public void setVersion(String version) {
+ this.version = version;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/BannerDTO.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/BannerDTO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/BannerDTO.java
new file mode 100644
index 0000000..c0a691b
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/BannerDTO.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.api.dto;
+
+import com.wordnik.swagger.annotations.ApiModelProperty;
+import javax.xml.bind.annotation.XmlType;
+
+/**
+ * Banners that should appear on the top and bottom of this NiFi.
+ */
+@XmlType(name = "banners")
+public class BannerDTO {
+
+ private String headerText;
+ private String footerText;
+
+ /* getters / setters */
+ /**
+ * The banner footer text.
+ *
+ * @return The footer text
+ */
+ @ApiModelProperty(
+ value = "The footer text."
+ )
+ public String getFooterText() {
+ return footerText;
+ }
+
+ public void setFooterText(String footerText) {
+ this.footerText = footerText;
+ }
+
+ /**
+ * The banner header text.
+ *
+ * @return The header text
+ */
+ @ApiModelProperty(
+ value = "The header text."
+ )
+ public String getHeaderText() {
+ return headerText;
+ }
+
+ public void setHeaderText(String headerText) {
+ this.headerText = headerText;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/BulletinBoardDTO.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/BulletinBoardDTO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/BulletinBoardDTO.java
new file mode 100644
index 0000000..455769a
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/BulletinBoardDTO.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.api.dto;
+
+import com.wordnik.swagger.annotations.ApiModelProperty;
+import java.util.Date;
+import java.util.List;
+
+import javax.xml.bind.annotation.XmlType;
+import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
+import org.apache.nifi.web.api.dto.util.TimeAdapter;
+
+/**
+ * The contents for the bulletin board including the bulletins and the timestamp when the board was generated.
+ */
+@XmlType(name = "bulletinBoard")
+public class BulletinBoardDTO {
+
+ private List<BulletinDTO> bulletins;
+ private Date generated;
+
+ /**
+ * @return bulletins to populate in the bulletin board
+ */
+ @ApiModelProperty(
+ value = "The bulletins in the bulletin board, that matches the supplied request."
+ )
+ public List<BulletinDTO> getBulletins() {
+ return bulletins;
+ }
+
+ public void setBulletins(List<BulletinDTO> bulletins) {
+ this.bulletins = bulletins;
+ }
+
+ /**
+ * @return when this bulletin board was generated
+ */
+ @XmlJavaTypeAdapter(TimeAdapter.class)
+ @ApiModelProperty(
+ value = "The timestamp when this report was generated."
+ )
+ public Date getGenerated() {
+ return generated;
+ }
+
+ public void setGenerated(final Date generated) {
+ this.generated = generated;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/BulletinDTO.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/BulletinDTO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/BulletinDTO.java
new file mode 100644
index 0000000..4c552f6
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/BulletinDTO.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.api.dto;
+
+import com.wordnik.swagger.annotations.ApiModelProperty;
+import java.util.Date;
+import javax.xml.bind.annotation.XmlType;
+import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
+import org.apache.nifi.web.api.dto.util.TimeAdapter;
+
+/**
+ * A bulletin that represents a notification about a passing event including, the source component (if applicable), the timestamp, the message, and where the bulletin originated (if applicable).
+ */
+@XmlType(name = "bulletin")
+public class BulletinDTO {
+
+ private Long id;
+ private String nodeAddress;
+ private String category;
+ private String groupId;
+ private String sourceId;
+ private String sourceName;
+ private String level;
+ private String message;
+ private Date timestamp;
+
+ /**
+ * @return id of this message
+ */
+ @ApiModelProperty(
+ value = "The id of the bulletin."
+ )
+ public Long getId() {
+ return id;
+ }
+
+ public void setId(Long id) {
+ this.id = id;
+ }
+
+ /**
+ * @return When clustered, the address of the node from which this bulletin originated
+ */
+ @ApiModelProperty(
+ value = "If clustered, the address of the node from whicih the bulletin originated."
+ )
+ public String getNodeAddress() {
+ return nodeAddress;
+ }
+
+ public void setNodeAddress(String nodeAddress) {
+ this.nodeAddress = nodeAddress;
+ }
+
+ /**
+ * @return group id of the source component
+ */
+ @ApiModelProperty(
+ value = "The group id of the source component."
+ )
+ public String getGroupId() {
+ return groupId;
+ }
+
+ public void setGroupId(String groupId) {
+ this.groupId = groupId;
+ }
+
+ /**
+ * @return category of this message
+ */
+ @ApiModelProperty(
+ value = "The catagory of this bulletin."
+ )
+ public String getCategory() {
+ return category;
+ }
+
+ public void setCategory(String category) {
+ this.category = category;
+ }
+
+ /**
+ * @return actual message
+ */
+ @ApiModelProperty(
+ value = "The bulletin message."
+ )
+ public String getMessage() {
+ return message;
+ }
+
+ public void setMessage(String message) {
+ this.message = message;
+ }
+
+ /**
+ * @return id of the source of this message
+ */
+ @ApiModelProperty(
+ value = "The id of the source component."
+ )
+ public String getSourceId() {
+ return sourceId;
+ }
+
+ public void setSourceId(String sourceId) {
+ this.sourceId = sourceId;
+ }
+
+ /**
+ * @return name of the source of this message
+ */
+ @ApiModelProperty(
+ value = "The name of the source component."
+ )
+ public String getSourceName() {
+ return sourceName;
+ }
+
+ public void setSourceName(String sourceName) {
+ this.sourceName = sourceName;
+ }
+
+ /**
+ * @return level of this bulletin
+ */
+ @ApiModelProperty(
+ value = "The level of the bulletin."
+ )
+ public String getLevel() {
+ return level;
+ }
+
+ public void setLevel(String level) {
+ this.level = level;
+ }
+
+ /**
+ * @return When this bulletin was generated as a formatted string
+ */
+ @XmlJavaTypeAdapter(TimeAdapter.class)
+ @ApiModelProperty(
+ value = "When this bulletin was generated."
+ )
+ public Date getTimestamp() {
+ return timestamp;
+ }
+
+ public void setTimestamp(Date timestamp) {
+ this.timestamp = timestamp;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/BulletinQueryDTO.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/BulletinQueryDTO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/BulletinQueryDTO.java
new file mode 100644
index 0000000..e47cdf9
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/BulletinQueryDTO.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.api.dto;
+
+import com.wordnik.swagger.annotations.ApiModelProperty;
+import javax.xml.bind.annotation.XmlType;
+
+/**
+ * A query for bulletin board. Will filter the resulting bulletin board according to the criteria in this query.
+ */
+@XmlType(name = "bulletinQuery")
+public class BulletinQueryDTO {
+
+ private String sourceId;
+ private String groupId;
+ private String name;
+ private String message;
+ private Long after;
+ private Integer limit;
+
+ /**
+ * @return Include bulletins after this id
+ */
+ @ApiModelProperty(
+ value = "Will include bulletins that occurred after this id. The ids are a one-up number that are used to ensure bulletins that "
+ + "occur at the same time will not be missed."
+ )
+ public Long getAfter() {
+ return after;
+ }
+
+ public void setAfter(Long after) {
+ this.after = after;
+ }
+
+ /**
+ * @return Include bulletin within this group. Supports a regular expression
+ */
+ @ApiModelProperty(
+ value = "Will include bulletins that occurred within this group. Supports a regular expression."
+ )
+ public String getGroupId() {
+ return groupId;
+ }
+
+ public void setGroupId(String groupId) {
+ this.groupId = groupId;
+ }
+
+ /**
+ * @return Include bulletins that match this message. Supports a regular expression
+ */
+ @ApiModelProperty(
+ value = "Will include bulletins that match this message. Supports a regular expression."
+ )
+ public String getMessage() {
+ return message;
+ }
+
+ public void setMessage(String message) {
+ this.message = message;
+ }
+
+ /**
+ * @return Include bulletins that match this name. Supports a regular expression
+ */
+ @ApiModelProperty(
+ value = "Will include bulletins that match this name. Supports a regular expression."
+ )
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ /**
+ * @return Include bulletins that match this id. Supports a source id
+ */
+ @ApiModelProperty(
+ value = "Will include bulletins from components that match this id. Supports a regular expression."
+ )
+ public String getSourceId() {
+ return sourceId;
+ }
+
+ public void setSourceId(String sourceId) {
+ this.sourceId = sourceId;
+ }
+
+ /**
+ * @return The maximum number of bulletins to return
+ */
+ @ApiModelProperty(
+ value = "The maximum number of bulletins to return."
+ )
+ public Integer getLimit() {
+ return limit;
+ }
+
+ public void setLimit(Integer limit) {
+ this.limit = limit;
+ }
+
+}
[27/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelInput.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelInput.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelInput.java
new file mode 100644
index 0000000..68a8dc4
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelInput.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.io.socket;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.channels.SocketChannel;
+
+import org.apache.nifi.stream.io.BufferedInputStream;
+import org.apache.nifi.stream.io.ByteCountingInputStream;
+import org.apache.nifi.remote.io.InterruptableInputStream;
+import org.apache.nifi.remote.protocol.CommunicationsInput;
+
+public class SocketChannelInput implements CommunicationsInput {
+
+ private final SocketChannelInputStream socketIn;
+ private final ByteCountingInputStream countingIn;
+ private final InputStream bufferedIn;
+ private final InterruptableInputStream interruptableIn;
+
+ public SocketChannelInput(final SocketChannel socketChannel) throws IOException {
+ this.socketIn = new SocketChannelInputStream(socketChannel);
+ countingIn = new ByteCountingInputStream(socketIn);
+ bufferedIn = new BufferedInputStream(countingIn);
+ interruptableIn = new InterruptableInputStream(bufferedIn);
+ }
+
+ @Override
+ public InputStream getInputStream() throws IOException {
+ return interruptableIn;
+ }
+
+ public void setTimeout(final int millis) {
+ socketIn.setTimeout(millis);
+ }
+
+ public boolean isDataAvailable() {
+ try {
+ return interruptableIn.available() > 0;
+ } catch (final Exception e) {
+ return false;
+ }
+ }
+
+ @Override
+ public long getBytesRead() {
+ return countingIn.getBytesRead();
+ }
+
+ public void interrupt() {
+ interruptableIn.interrupt();
+ }
+
+ @Override
+ public void consume() throws IOException {
+ socketIn.consume();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelOutput.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelOutput.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelOutput.java
new file mode 100644
index 0000000..13974a5
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelOutput.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.io.socket;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.channels.SocketChannel;
+
+import org.apache.nifi.stream.io.BufferedOutputStream;
+import org.apache.nifi.stream.io.ByteCountingOutputStream;
+import org.apache.nifi.remote.io.InterruptableOutputStream;
+import org.apache.nifi.remote.protocol.CommunicationsOutput;
+
+public class SocketChannelOutput implements CommunicationsOutput {
+
+ private final SocketChannelOutputStream socketOutStream;
+ private final ByteCountingOutputStream countingOut;
+ private final OutputStream bufferedOut;
+ private final InterruptableOutputStream interruptableOut;
+
+ public SocketChannelOutput(final SocketChannel socketChannel) throws IOException {
+ socketOutStream = new SocketChannelOutputStream(socketChannel);
+ countingOut = new ByteCountingOutputStream(socketOutStream);
+ bufferedOut = new BufferedOutputStream(countingOut);
+ interruptableOut = new InterruptableOutputStream(bufferedOut);
+ }
+
+ @Override
+ public OutputStream getOutputStream() throws IOException {
+ return interruptableOut;
+ }
+
+ public void setTimeout(final int timeout) {
+ socketOutStream.setTimeout(timeout);
+ }
+
+ @Override
+ public long getBytesWritten() {
+ return countingOut.getBytesWritten();
+ }
+
+ public void interrupt() {
+ interruptableOut.interrupt();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelCommunicationsSession.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelCommunicationsSession.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelCommunicationsSession.java
new file mode 100644
index 0000000..5e5abc7
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelCommunicationsSession.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.io.socket.ssl;
+
+import java.io.IOException;
+
+import org.apache.nifi.remote.AbstractCommunicationsSession;
+
+public class SSLSocketChannelCommunicationsSession extends AbstractCommunicationsSession {
+
+ private final SSLSocketChannel channel;
+ private final SSLSocketChannelInput request;
+ private final SSLSocketChannelOutput response;
+
+ public SSLSocketChannelCommunicationsSession(final SSLSocketChannel channel, final String uri) {
+ super(uri);
+ request = new SSLSocketChannelInput(channel);
+ response = new SSLSocketChannelOutput(channel);
+ this.channel = channel;
+ }
+
+ @Override
+ public SSLSocketChannelInput getInput() {
+ return request;
+ }
+
+ @Override
+ public SSLSocketChannelOutput getOutput() {
+ return response;
+ }
+
+ @Override
+ public void setTimeout(final int millis) throws IOException {
+ channel.setTimeout(millis);
+ }
+
+ @Override
+ public int getTimeout() throws IOException {
+ return channel.getTimeout();
+ }
+
+ @Override
+ public void close() throws IOException {
+ IOException suppressed = null;
+
+ try {
+ request.consume();
+ } catch (final IOException ioe) {
+ suppressed = ioe;
+ }
+
+ try {
+ channel.close();
+ } catch (final IOException ioe) {
+ if (suppressed != null) {
+ ioe.addSuppressed(suppressed);
+ }
+
+ throw ioe;
+ }
+
+ if (suppressed != null) {
+ throw suppressed;
+ }
+ }
+
+ @Override
+ public boolean isClosed() {
+ return channel.isClosed();
+ }
+
+ @Override
+ public boolean isDataAvailable() {
+ try {
+ return request.isDataAvailable();
+ } catch (final Exception e) {
+ return false;
+ }
+ }
+
+ @Override
+ public long getBytesWritten() {
+ return response.getBytesWritten();
+ }
+
+ @Override
+ public long getBytesRead() {
+ return request.getBytesRead();
+ }
+
+ @Override
+ public void interrupt() {
+ channel.interrupt();
+ }
+
+ @Override
+ public String toString() {
+ return super.toString() + "[SSLSocketChannel=" + channel + "]";
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelInput.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelInput.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelInput.java
new file mode 100644
index 0000000..6cd2344
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelInput.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.io.socket.ssl;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.nifi.stream.io.BufferedInputStream;
+import org.apache.nifi.stream.io.ByteCountingInputStream;
+import org.apache.nifi.remote.protocol.CommunicationsInput;
+
+public class SSLSocketChannelInput implements CommunicationsInput {
+
+ private final SSLSocketChannelInputStream in;
+ private final ByteCountingInputStream countingIn;
+ private final InputStream bufferedIn;
+
+ public SSLSocketChannelInput(final SSLSocketChannel socketChannel) {
+ in = new SSLSocketChannelInputStream(socketChannel);
+ countingIn = new ByteCountingInputStream(in);
+ this.bufferedIn = new BufferedInputStream(countingIn);
+ }
+
+ @Override
+ public InputStream getInputStream() throws IOException {
+ return bufferedIn;
+ }
+
+ public boolean isDataAvailable() throws IOException {
+ return bufferedIn.available() > 0;
+ }
+
+ @Override
+ public long getBytesRead() {
+ return countingIn.getBytesRead();
+ }
+
+ @Override
+ public void consume() throws IOException {
+ in.consume();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelOutput.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelOutput.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelOutput.java
new file mode 100644
index 0000000..33d13cb
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelOutput.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.io.socket.ssl;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+import org.apache.nifi.stream.io.BufferedOutputStream;
+import org.apache.nifi.stream.io.ByteCountingOutputStream;
+import org.apache.nifi.remote.protocol.CommunicationsOutput;
+
+public class SSLSocketChannelOutput implements CommunicationsOutput {
+
+ private final OutputStream out;
+ private final ByteCountingOutputStream countingOut;
+
+ public SSLSocketChannelOutput(final SSLSocketChannel channel) {
+ countingOut = new ByteCountingOutputStream(new SSLSocketChannelOutputStream(channel));
+ out = new BufferedOutputStream(countingOut);
+ }
+
+ @Override
+ public OutputStream getOutputStream() throws IOException {
+ return out;
+ }
+
+ @Override
+ public long getBytesWritten() {
+ return countingOut.getBytesWritten();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/ClientProtocol.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/ClientProtocol.java
new file mode 100644
index 0000000..2efea11
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/ClientProtocol.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.protocol;
+
+import java.io.IOException;
+import java.util.Set;
+
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.remote.Peer;
+import org.apache.nifi.remote.PeerStatus;
+import org.apache.nifi.remote.Transaction;
+import org.apache.nifi.remote.TransferDirection;
+import org.apache.nifi.remote.VersionedRemoteResource;
+import org.apache.nifi.remote.codec.FlowFileCodec;
+import org.apache.nifi.remote.exception.HandshakeException;
+import org.apache.nifi.remote.exception.PortNotRunningException;
+import org.apache.nifi.remote.exception.ProtocolException;
+import org.apache.nifi.remote.exception.UnknownPortException;
+
+public interface ClientProtocol extends VersionedRemoteResource {
+
+ void handshake(Peer peer) throws IOException, HandshakeException, UnknownPortException, PortNotRunningException;
+
+ Set<PeerStatus> getPeerStatuses(Peer currentPeer) throws IOException, ProtocolException;
+
+ FlowFileCodec negotiateCodec(Peer peer) throws IOException, ProtocolException;
+
+ int receiveFlowFiles(Peer peer, ProcessContext context, ProcessSession session, FlowFileCodec codec) throws IOException, ProtocolException;
+
+ int transferFlowFiles(Peer peer, ProcessContext context, ProcessSession session, FlowFileCodec codec) throws IOException, ProtocolException;
+
+ void shutdown(Peer peer) throws IOException, ProtocolException;
+
+ boolean isReadyForFileTransfer();
+
+ Transaction startTransaction(Peer peer, FlowFileCodec codec, TransferDirection direction) throws IOException;
+
+ /**
+ * @return <code>true</code> if remote instance indicates that the port is
+ * invalid
+ * @throws IllegalStateException if a handshake has not successfully
+ * completed
+ */
+ boolean isPortInvalid() throws IllegalStateException;
+
+ /**
+ * @return <code>true</code> if remote instance indicates that the port is
+ * unknown
+ * @throws IllegalStateException if a handshake has not successfully
+ * completed
+ */
+ boolean isPortUnknown();
+
+ /**
+ * @return <code>true</code> if remote instance indicates that the port's
+ * destination is full
+ * @throws IllegalStateException if a handshake has not successfully
+ * completed
+ */
+ boolean isDestinationFull();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/CommunicationsInput.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/CommunicationsInput.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/CommunicationsInput.java
new file mode 100644
index 0000000..3fa3e96
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/CommunicationsInput.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.protocol;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+public interface CommunicationsInput {
+
+ /**
+ * Reads all data currently on the socket and throws it away
+ *
+ * @throws IOException if unable to consume
+ */
+ void consume() throws IOException;
+
+ InputStream getInputStream() throws IOException;
+
+ long getBytesRead();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/CommunicationsOutput.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/CommunicationsOutput.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/CommunicationsOutput.java
new file mode 100644
index 0000000..95cab29
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/CommunicationsOutput.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.protocol;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+public interface CommunicationsOutput {
+
+ OutputStream getOutputStream() throws IOException;
+
+ long getBytesWritten();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/CommunicationsSession.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/CommunicationsSession.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/CommunicationsSession.java
new file mode 100644
index 0000000..aff73ba
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/CommunicationsSession.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.protocol;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+public interface CommunicationsSession extends Closeable {
+
+ public static final byte[] MAGIC_BYTES = {(byte) 'N', (byte) 'i', (byte) 'F', (byte) 'i'};
+
+ CommunicationsInput getInput();
+
+ CommunicationsOutput getOutput();
+
+ void setTimeout(int millis) throws IOException;
+
+ int getTimeout() throws IOException;
+
+ void setUri(String uri);
+
+ String getUri();
+
+ String getUserDn();
+
+ void setUserDn(String dn);
+
+ boolean isDataAvailable();
+
+ long getBytesWritten();
+
+ long getBytesRead();
+
+ /**
+ * Asynchronously interrupts this FlowFileCodec. Implementations must ensure
+ * that they stop sending and receiving data as soon as possible after this
+ * method has been called, even if doing so results in sending only partial
+ * data to the peer. This will usually result in the peer throwing a
+ * SocketTimeoutException.
+ */
+ void interrupt();
+
+ /**
+ * @return <code>true</code> if the connection is closed, <code>false</code>
+ * otherwise
+ */
+ boolean isClosed();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/DataPacket.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/DataPacket.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/DataPacket.java
new file mode 100644
index 0000000..e7b6d06
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/DataPacket.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.protocol;
+
+import java.io.InputStream;
+import java.util.Map;
+
+/**
+ * Represents a piece of data that is to be sent to or that was received from a
+ * NiFi instance.
+ */
+public interface DataPacket {
+
+ /**
+ * The key-value attributes that are to be associated with the data
+ *
+ * @return all attributes
+ */
+ Map<String, String> getAttributes();
+
+ /**
+ * An InputStream from which the content can be read
+ *
+ * @return input stream to the data
+ */
+ InputStream getData();
+
+ /**
+ * The length of the InputStream.
+ *
+ * @return length of the inputstream.
+ */
+ long getSize();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/RequestType.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/RequestType.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/RequestType.java
new file mode 100644
index 0000000..41334fe
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/RequestType.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.protocol;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+public enum RequestType {
+
+ NEGOTIATE_FLOWFILE_CODEC,
+ REQUEST_PEER_LIST,
+ SEND_FLOWFILES,
+ RECEIVE_FLOWFILES,
+ SHUTDOWN;
+
+ public void writeRequestType(final DataOutputStream dos) throws IOException {
+ dos.writeUTF(name());
+ }
+
+ public static RequestType readRequestType(final DataInputStream dis) throws IOException {
+ final String requestTypeVal = dis.readUTF();
+ try {
+ return RequestType.valueOf(requestTypeVal);
+ } catch (final Exception e) {
+ throw new IOException("Could not determine RequestType: received invalid value " + requestTypeVal);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/socket/HandshakeProperty.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/socket/HandshakeProperty.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/socket/HandshakeProperty.java
new file mode 100644
index 0000000..016690c
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/socket/HandshakeProperty.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.protocol.socket;
+
+/**
+ * Enumeration of Properties that can be used for the Site-to-Site Socket
+ * Protocol.
+ */
+public enum HandshakeProperty {
+
+ /**
+ * Boolean value indicating whether or not the contents of a FlowFile should
+ * be GZipped when transferred.
+ */
+ GZIP,
+ /**
+ * The unique identifier of the port to communicate with
+ */
+ PORT_IDENTIFIER,
+ /**
+ * Indicates the number of milliseconds after the request was made that the
+ * client will wait for a response. If no response has been received by the
+ * time this value expires, the server can move on without attempting to
+ * service the request because the client will have already disconnected.
+ */
+ REQUEST_EXPIRATION_MILLIS,
+ /**
+ * The preferred number of FlowFiles that the server should send to the
+ * client when pulling data. This property was introduced in version 5 of
+ * the protocol.
+ */
+ BATCH_COUNT,
+ /**
+ * The preferred number of bytes that the server should send to the client
+ * when pulling data. This property was introduced in version 5 of the
+ * protocol.
+ */
+ BATCH_SIZE,
+ /**
+ * The preferred amount of time that the server should send data to the
+ * client when pulling data. This property was introduced in version 5 of
+ * the protocol. Value is in milliseconds.
+ */
+ BATCH_DURATION;
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/socket/Response.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/socket/Response.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/socket/Response.java
new file mode 100644
index 0000000..6ad2ba0
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/socket/Response.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.protocol.socket;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+
+import org.apache.nifi.remote.exception.ProtocolException;
+
+public class Response {
+
+ private final ResponseCode code;
+ private final String message;
+
+ private Response(final ResponseCode code, final String explanation) {
+ this.code = code;
+ this.message = explanation;
+ }
+
+ public ResponseCode getCode() {
+ return code;
+ }
+
+ public String getMessage() {
+ return message;
+ }
+
+ public static Response read(final DataInputStream in) throws IOException, ProtocolException {
+ final ResponseCode code = ResponseCode.readCode(in);
+ final String message = code.containsMessage() ? in.readUTF() : null;
+ return new Response(code, message);
+ }
+
+ @Override
+ public String toString() {
+ return code + ": " + message;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/socket/ResponseCode.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/socket/ResponseCode.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/socket/ResponseCode.java
new file mode 100644
index 0000000..0e1359e
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/socket/ResponseCode.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.protocol.socket;
+
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.nifi.remote.exception.ProtocolException;
+
+public enum ResponseCode {
+
+ RESERVED(0, "Reserved for Future Use", false), // This will likely be used if we ever need to expand the length of
+ // ResponseCode, so that we can indicate a 0 followed by some other bytes
+
+ // handshaking properties
+ PROPERTIES_OK(1, "Properties OK", false),
+ UNKNOWN_PROPERTY_NAME(230, "Unknown Property Name", true),
+ ILLEGAL_PROPERTY_VALUE(231, "Illegal Property Value", true),
+ MISSING_PROPERTY(232, "Missing Property", true),
+ // transaction indicators
+ CONTINUE_TRANSACTION(10, "Continue Transaction", false),
+ FINISH_TRANSACTION(11, "Finish Transaction", false),
+ CONFIRM_TRANSACTION(12, "Confirm Transaction", true), // "Explanation" of this code is the checksum
+ TRANSACTION_FINISHED(13, "Transaction Finished", false),
+ TRANSACTION_FINISHED_BUT_DESTINATION_FULL(14, "Transaction Finished But Destination is Full", false),
+ CANCEL_TRANSACTION(15, "Cancel Transaction", true),
+ BAD_CHECKSUM(19, "Bad Checksum", false),
+ // data availability indicators
+ MORE_DATA(20, "More Data Exists", false),
+ NO_MORE_DATA(21, "No More Data Exists", false),
+ // port state indicators
+ UNKNOWN_PORT(200, "Unknown Port", false),
+ PORT_NOT_IN_VALID_STATE(201, "Port Not in a Valid State", true),
+ PORTS_DESTINATION_FULL(202, "Port's Destination is Full", false),
+ // authorization
+ UNAUTHORIZED(240, "User Not Authorized", true),
+ // error indicators
+ ABORT(250, "Abort", true),
+ UNRECOGNIZED_RESPONSE_CODE(254, "Unrecognized Response Code", false),
+ END_OF_STREAM(255, "End of Stream", false);
+
+ private static final ResponseCode[] codeArray = new ResponseCode[256];
+
+ static {
+ for (final ResponseCode responseCode : ResponseCode.values()) {
+ codeArray[responseCode.getCode()] = responseCode;
+ }
+ }
+
+ private static final byte CODE_SEQUENCE_VALUE_1 = (byte) 'R';
+ private static final byte CODE_SEQUENCE_VALUE_2 = (byte) 'C';
+ private final int code;
+ private final byte[] codeSequence;
+ private final String description;
+ private final boolean containsMessage;
+
+ private ResponseCode(final int code, final String description, final boolean containsMessage) {
+ this.codeSequence = new byte[]{CODE_SEQUENCE_VALUE_1, CODE_SEQUENCE_VALUE_2, (byte) code};
+ this.code = code;
+ this.description = description;
+ this.containsMessage = containsMessage;
+ }
+
+ public int getCode() {
+ return code;
+ }
+
+ public byte[] getCodeSequence() {
+ return codeSequence;
+ }
+
+ @Override
+ public String toString() {
+ return description;
+ }
+
+ public boolean containsMessage() {
+ return containsMessage;
+ }
+
+ public void writeResponse(final DataOutputStream out) throws IOException {
+ if (containsMessage()) {
+ throw new IllegalArgumentException("ResponseCode " + code + " expects an explanation");
+ }
+
+ out.write(getCodeSequence());
+ out.flush();
+ }
+
+ public void writeResponse(final DataOutputStream out, final String explanation) throws IOException {
+ if (!containsMessage()) {
+ throw new IllegalArgumentException("ResponseCode " + code + " does not expect an explanation");
+ }
+
+ out.write(getCodeSequence());
+ out.writeUTF(explanation);
+ out.flush();
+ }
+
+ static ResponseCode readCode(final InputStream in) throws IOException, ProtocolException {
+ final int byte1 = in.read();
+ if (byte1 < 0) {
+ throw new EOFException();
+ } else if (byte1 != CODE_SEQUENCE_VALUE_1) {
+ throw new ProtocolException("Expected to receive ResponseCode, but the stream did not have a ResponseCode");
+ }
+
+ final int byte2 = in.read();
+ if (byte2 < 0) {
+ throw new EOFException();
+ } else if (byte2 != CODE_SEQUENCE_VALUE_2) {
+ throw new ProtocolException("Expected to receive ResponseCode, but the stream did not have a ResponseCode");
+ }
+
+ final int byte3 = in.read();
+ if (byte3 < 0) {
+ throw new EOFException();
+ }
+
+ final ResponseCode responseCode = codeArray[byte3];
+ if (responseCode == null) {
+ throw new ProtocolException("Received Response Code of " + byte3 + " but do not recognize this code");
+ }
+ return responseCode;
+ }
+
+ public static ResponseCode fromSequence(final byte[] value) {
+ final int code = value[3] & 0xFF;
+ final ResponseCode responseCode = codeArray[code];
+ return (responseCode == null) ? UNRECOGNIZED_RESPONSE_CODE : responseCode;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/socket/SocketClientProtocol.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/socket/SocketClientProtocol.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/socket/SocketClientProtocol.java
new file mode 100644
index 0000000..de845ee
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/socket/SocketClientProtocol.java
@@ -0,0 +1,446 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.protocol.socket;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.nifi.events.EventReporter;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.io.InputStreamCallback;
+import org.apache.nifi.remote.Peer;
+import org.apache.nifi.remote.PeerDescription;
+import org.apache.nifi.remote.PeerStatus;
+import org.apache.nifi.remote.RemoteDestination;
+import org.apache.nifi.remote.RemoteResourceInitiator;
+import org.apache.nifi.remote.StandardVersionNegotiator;
+import org.apache.nifi.remote.Transaction;
+import org.apache.nifi.remote.TransferDirection;
+import org.apache.nifi.remote.VersionNegotiator;
+import org.apache.nifi.remote.codec.FlowFileCodec;
+import org.apache.nifi.remote.codec.StandardFlowFileCodec;
+import org.apache.nifi.remote.exception.HandshakeException;
+import org.apache.nifi.remote.exception.ProtocolException;
+import org.apache.nifi.remote.protocol.ClientProtocol;
+import org.apache.nifi.remote.protocol.CommunicationsSession;
+import org.apache.nifi.remote.protocol.DataPacket;
+import org.apache.nifi.remote.protocol.RequestType;
+import org.apache.nifi.remote.util.StandardDataPacket;
+import org.apache.nifi.util.FormatUtils;
+import org.apache.nifi.util.StopWatch;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class SocketClientProtocol implements ClientProtocol {
+
+ private final VersionNegotiator versionNegotiator = new StandardVersionNegotiator(5, 4, 3, 2, 1);
+
+ private RemoteDestination destination;
+ private boolean useCompression = false;
+
+ private String commsIdentifier;
+ private boolean handshakeComplete = false;
+
+ private final Logger logger = LoggerFactory.getLogger(SocketClientProtocol.class);
+
+ private Response handshakeResponse = null;
+ private boolean readyForFileTransfer = false;
+ private String transitUriPrefix = null;
+ private int timeoutMillis = 30000;
+
+ private int batchCount;
+ private long batchSize;
+ private long batchMillis;
+ private EventReporter eventReporter;
+
+ private static final long BATCH_SEND_NANOS = TimeUnit.SECONDS.toNanos(5L); // send batches of up to 5 seconds
+
+ public SocketClientProtocol() {
+ }
+
+ public void setPreferredBatchCount(final int count) {
+ this.batchCount = count;
+ }
+
+ public void setPreferredBatchSize(final long bytes) {
+ this.batchSize = bytes;
+ }
+
+ public void setPreferredBatchDuration(final long millis) {
+ this.batchMillis = millis;
+ }
+
+ public void setEventReporter(final EventReporter eventReporter) {
+ this.eventReporter = eventReporter;
+ }
+
+ public void setDestination(final RemoteDestination destination) {
+ this.destination = destination;
+ this.useCompression = destination.isUseCompression();
+ }
+
+ public void setTimeout(final int timeoutMillis) {
+ this.timeoutMillis = timeoutMillis;
+ }
+
+ @Override
+ public void handshake(final Peer peer) throws IOException, HandshakeException {
+ handshake(peer, destination.getIdentifier());
+ }
+
+ public void handshake(final Peer peer, final String destinationId) throws IOException, HandshakeException {
+ if (handshakeComplete) {
+ throw new IllegalStateException("Handshake has already been completed");
+ }
+ commsIdentifier = UUID.randomUUID().toString();
+ logger.debug("{} handshaking with {}", this, peer);
+
+ final Map<HandshakeProperty, String> properties = new HashMap<>();
+ properties.put(HandshakeProperty.GZIP, String.valueOf(useCompression));
+
+ if (destinationId != null) {
+ properties.put(HandshakeProperty.PORT_IDENTIFIER, destinationId);
+ }
+
+ properties.put(HandshakeProperty.REQUEST_EXPIRATION_MILLIS, String.valueOf(timeoutMillis));
+
+ if (versionNegotiator.getVersion() >= 5) {
+ if (batchCount > 0) {
+ properties.put(HandshakeProperty.BATCH_COUNT, String.valueOf(batchCount));
+ }
+ if (batchSize > 0L) {
+ properties.put(HandshakeProperty.BATCH_SIZE, String.valueOf(batchSize));
+ }
+ if (batchMillis > 0L) {
+ properties.put(HandshakeProperty.BATCH_DURATION, String.valueOf(batchMillis));
+ }
+ }
+
+ final CommunicationsSession commsSession = peer.getCommunicationsSession();
+ commsSession.setTimeout(timeoutMillis);
+ final DataInputStream dis = new DataInputStream(commsSession.getInput().getInputStream());
+ final DataOutputStream dos = new DataOutputStream(commsSession.getOutput().getOutputStream());
+
+ dos.writeUTF(commsIdentifier);
+
+ if (versionNegotiator.getVersion() >= 3) {
+ dos.writeUTF(peer.getUrl());
+ transitUriPrefix = peer.getUrl();
+
+ if (!transitUriPrefix.endsWith("/")) {
+ transitUriPrefix = transitUriPrefix + "/";
+ }
+ }
+
+ logger.debug("Handshaking with properties {}", properties);
+ dos.writeInt(properties.size());
+ for (final Map.Entry<HandshakeProperty, String> entry : properties.entrySet()) {
+ dos.writeUTF(entry.getKey().name());
+ dos.writeUTF(entry.getValue());
+ }
+
+ dos.flush();
+
+ try {
+ handshakeResponse = Response.read(dis);
+ } catch (final ProtocolException e) {
+ throw new HandshakeException(e);
+ }
+
+ switch (handshakeResponse.getCode()) {
+ case PORT_NOT_IN_VALID_STATE:
+ case UNKNOWN_PORT:
+ case PORTS_DESTINATION_FULL:
+ break;
+ case PROPERTIES_OK:
+ readyForFileTransfer = true;
+ break;
+ default:
+ logger.error("{} received unexpected response {} from {} when negotiating Codec", new Object[]{
+ this, handshakeResponse, peer});
+ peer.close();
+ throw new HandshakeException("Received unexpected response " + handshakeResponse);
+ }
+
+ logger.debug("{} Finished handshake with {}", this, peer);
+ handshakeComplete = true;
+ }
+
+ @Override
+ public boolean isReadyForFileTransfer() {
+ return readyForFileTransfer;
+ }
+
+ @Override
+ public boolean isPortInvalid() {
+ if (!handshakeComplete) {
+ throw new IllegalStateException("Handshake has not completed successfully");
+ }
+ return handshakeResponse.getCode() == ResponseCode.PORT_NOT_IN_VALID_STATE;
+ }
+
+ @Override
+ public boolean isPortUnknown() {
+ if (!handshakeComplete) {
+ throw new IllegalStateException("Handshake has not completed successfully");
+ }
+ return handshakeResponse.getCode() == ResponseCode.UNKNOWN_PORT;
+ }
+
+ @Override
+ public boolean isDestinationFull() {
+ if (!handshakeComplete) {
+ throw new IllegalStateException("Handshake has not completed successfully");
+ }
+ return handshakeResponse.getCode() == ResponseCode.PORTS_DESTINATION_FULL;
+ }
+
+ @Override
+ public Set<PeerStatus> getPeerStatuses(final Peer peer) throws IOException {
+ if (!handshakeComplete) {
+ throw new IllegalStateException("Handshake has not been performed");
+ }
+
+ logger.debug("{} Get Peer Statuses from {}", this, peer);
+ final CommunicationsSession commsSession = peer.getCommunicationsSession();
+ final DataInputStream dis = new DataInputStream(commsSession.getInput().getInputStream());
+ final DataOutputStream dos = new DataOutputStream(commsSession.getOutput().getOutputStream());
+
+ RequestType.REQUEST_PEER_LIST.writeRequestType(dos);
+ dos.flush();
+ final int numPeers = dis.readInt();
+ final Set<PeerStatus> peers = new HashSet<>(numPeers);
+ for (int i = 0; i < numPeers; i++) {
+ final String hostname = dis.readUTF();
+ final int port = dis.readInt();
+ final boolean secure = dis.readBoolean();
+ final int flowFileCount = dis.readInt();
+ peers.add(new PeerStatus(new PeerDescription(hostname, port, secure), flowFileCount));
+ }
+
+ logger.debug("{} Received {} Peer Statuses from {}", this, peers.size(), peer);
+ return peers;
+ }
+
+ @Override
+ public FlowFileCodec negotiateCodec(final Peer peer) throws IOException, ProtocolException {
+ if (!handshakeComplete) {
+ throw new IllegalStateException("Handshake has not been performed");
+ }
+
+ logger.debug("{} Negotiating Codec with {}", this, peer);
+ final CommunicationsSession commsSession = peer.getCommunicationsSession();
+ final DataInputStream dis = new DataInputStream(commsSession.getInput().getInputStream());
+ final DataOutputStream dos = new DataOutputStream(commsSession.getOutput().getOutputStream());
+
+ RequestType.NEGOTIATE_FLOWFILE_CODEC.writeRequestType(dos);
+
+ FlowFileCodec codec = new StandardFlowFileCodec();
+ try {
+ codec = (FlowFileCodec) RemoteResourceInitiator.initiateResourceNegotiation(codec, dis, dos);
+ } catch (HandshakeException e) {
+ throw new ProtocolException(e.toString());
+ }
+ logger.debug("{} negotiated FlowFileCodec {} with {}", new Object[]{this, codec, commsSession});
+
+ return codec;
+ }
+
+ @Override
+ public Transaction startTransaction(final Peer peer, final FlowFileCodec codec, final TransferDirection direction) throws IOException, ProtocolException {
+ if (!handshakeComplete) {
+ throw new IllegalStateException("Handshake has not been performed");
+ }
+ if (!readyForFileTransfer) {
+ throw new IllegalStateException("Cannot start transaction; handshake resolution was " + handshakeResponse);
+ }
+
+ return new SocketClientTransaction(versionNegotiator.getVersion(), destination.getIdentifier(), peer, codec,
+ direction, useCompression, (int) destination.getYieldPeriod(TimeUnit.MILLISECONDS), eventReporter);
+ }
+
+ @Override
+ public int receiveFlowFiles(final Peer peer, final ProcessContext context, final ProcessSession session, final FlowFileCodec codec) throws IOException, ProtocolException {
+ final String userDn = peer.getCommunicationsSession().getUserDn();
+ final Transaction transaction = startTransaction(peer, codec, TransferDirection.RECEIVE);
+
+ final StopWatch stopWatch = new StopWatch(true);
+ final Set<FlowFile> flowFilesReceived = new HashSet<>();
+ long bytesReceived = 0L;
+
+ while (true) {
+ final long start = System.nanoTime();
+ final DataPacket dataPacket = transaction.receive();
+ if (dataPacket == null) {
+ if (flowFilesReceived.isEmpty()) {
+ peer.penalize(destination.getIdentifier(), destination.getYieldPeriod(TimeUnit.MILLISECONDS));
+ }
+ break;
+ }
+
+ FlowFile flowFile = session.create();
+ flowFile = session.putAllAttributes(flowFile, dataPacket.getAttributes());
+ flowFile = session.importFrom(dataPacket.getData(), flowFile);
+ final long receiveNanos = System.nanoTime() - start;
+
+ String sourceFlowFileIdentifier = dataPacket.getAttributes().get(CoreAttributes.UUID.key());
+ if (sourceFlowFileIdentifier == null) {
+ sourceFlowFileIdentifier = "<Unknown Identifier>";
+ }
+
+ final String transitUri = (transitUriPrefix == null) ? peer.getUrl() : transitUriPrefix + sourceFlowFileIdentifier;
+ session.getProvenanceReporter().receive(flowFile, transitUri, "urn:nifi:" + sourceFlowFileIdentifier, "Remote Host="
+ + peer.getHost() + ", Remote DN=" + userDn, TimeUnit.NANOSECONDS.toMillis(receiveNanos));
+
+ session.transfer(flowFile, Relationship.ANONYMOUS);
+ bytesReceived += dataPacket.getSize();
+ }
+
+ // Confirm that what we received was the correct data.
+ transaction.confirm();
+
+ // Commit the session so that we have persisted the data
+ session.commit();
+
+ transaction.complete();
+ logger.debug("{} Sending TRANSACTION_FINISHED_BUT_DESTINATION_FULL to {}", this, peer);
+
+ if (!flowFilesReceived.isEmpty()) {
+ stopWatch.stop();
+ final String flowFileDescription = flowFilesReceived.size() < 20 ? flowFilesReceived.toString() : flowFilesReceived.size() + " FlowFiles";
+ final String uploadDataRate = stopWatch.calculateDataRate(bytesReceived);
+ final long uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
+ final String dataSize = FormatUtils.formatDataSize(bytesReceived);
+ logger.info("{} Successfully receveied {} ({}) from {} in {} milliseconds at a rate of {}", new Object[]{
+ this, flowFileDescription, dataSize, peer, uploadMillis, uploadDataRate});
+ }
+
+ return flowFilesReceived.size();
+ }
+
+ @Override
+ public int transferFlowFiles(final Peer peer, final ProcessContext context, final ProcessSession session, final FlowFileCodec codec) throws IOException, ProtocolException {
+ FlowFile flowFile = session.get();
+ if (flowFile == null) {
+ return 0;
+ }
+
+ try {
+ final String userDn = peer.getCommunicationsSession().getUserDn();
+ final long startSendingNanos = System.nanoTime();
+ final StopWatch stopWatch = new StopWatch(true);
+ long bytesSent = 0L;
+
+ final Transaction transaction = startTransaction(peer, codec, TransferDirection.SEND);
+
+ final Set<FlowFile> flowFilesSent = new HashSet<>();
+ boolean continueTransaction = true;
+ while (continueTransaction) {
+ final long startNanos = System.nanoTime();
+ // call codec.encode within a session callback so that we have the InputStream to read the FlowFile
+ final FlowFile toWrap = flowFile;
+ session.read(flowFile, new InputStreamCallback() {
+ @Override
+ public void process(final InputStream in) throws IOException {
+ final DataPacket dataPacket = new StandardDataPacket(toWrap.getAttributes(), in, toWrap.getSize());
+ transaction.send(dataPacket);
+ }
+ });
+
+ final long transferNanos = System.nanoTime() - startNanos;
+ final long transferMillis = TimeUnit.MILLISECONDS.convert(transferNanos, TimeUnit.NANOSECONDS);
+
+ flowFilesSent.add(flowFile);
+ bytesSent += flowFile.getSize();
+ logger.debug("{} Sent {} to {}", this, flowFile, peer);
+
+ final String transitUri = (transitUriPrefix == null) ? peer.getUrl() : transitUriPrefix + flowFile.getAttribute(CoreAttributes.UUID.key());
+ session.getProvenanceReporter().send(flowFile, transitUri, "Remote Host=" + peer.getHost() + ", Remote DN=" + userDn, transferMillis, false);
+ session.remove(flowFile);
+
+ final long sendingNanos = System.nanoTime() - startSendingNanos;
+ if (sendingNanos < BATCH_SEND_NANOS) {
+ flowFile = session.get();
+ } else {
+ flowFile = null;
+ }
+
+ continueTransaction = (flowFile != null);
+ }
+
+ transaction.confirm();
+
+ // consume input stream entirely, ignoring its contents. If we
+ // don't do this, the Connection will not be returned to the pool
+ stopWatch.stop();
+ final String uploadDataRate = stopWatch.calculateDataRate(bytesSent);
+ final long uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
+ final String dataSize = FormatUtils.formatDataSize(bytesSent);
+
+ session.commit();
+ transaction.complete();
+
+ final String flowFileDescription = (flowFilesSent.size() < 20) ? flowFilesSent.toString() : flowFilesSent.size() + " FlowFiles";
+ logger.info("{} Successfully sent {} ({}) to {} in {} milliseconds at a rate of {}", new Object[]{
+ this, flowFileDescription, dataSize, peer, uploadMillis, uploadDataRate});
+
+ return flowFilesSent.size();
+ } catch (final Exception e) {
+ session.rollback();
+ throw e;
+ }
+ }
+
+ @Override
+ public VersionNegotiator getVersionNegotiator() {
+ return versionNegotiator;
+ }
+
+ @Override
+ public void shutdown(final Peer peer) throws IOException {
+ readyForFileTransfer = false;
+ final CommunicationsSession commsSession = peer.getCommunicationsSession();
+ final DataOutputStream dos = new DataOutputStream(commsSession.getOutput().getOutputStream());
+
+ logger.debug("{} Shutting down with {}", this, peer);
+ // Indicate that we would like to have some data
+ RequestType.SHUTDOWN.writeRequestType(dos);
+ dos.flush();
+ }
+
+ @Override
+ public String getResourceName() {
+ return "SocketFlowFileProtocol";
+ }
+
+ @Override
+ public String toString() {
+ return "SocketClientProtocol[CommsID=" + commsIdentifier + "]";
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/socket/SocketClientTransaction.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/socket/SocketClientTransaction.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/socket/SocketClientTransaction.java
new file mode 100644
index 0000000..e83ea28
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/socket/SocketClientTransaction.java
@@ -0,0 +1,406 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.protocol.socket;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Map;
+import java.util.zip.CRC32;
+import java.util.zip.CheckedInputStream;
+import java.util.zip.CheckedOutputStream;
+
+import org.apache.nifi.events.EventReporter;
+import org.apache.nifi.remote.Communicant;
+import org.apache.nifi.remote.Peer;
+import org.apache.nifi.remote.Transaction;
+import org.apache.nifi.remote.TransactionCompletion;
+import org.apache.nifi.remote.TransferDirection;
+import org.apache.nifi.remote.codec.FlowFileCodec;
+import org.apache.nifi.remote.exception.ProtocolException;
+import org.apache.nifi.remote.io.CompressionInputStream;
+import org.apache.nifi.remote.io.CompressionOutputStream;
+import org.apache.nifi.remote.protocol.DataPacket;
+import org.apache.nifi.remote.protocol.RequestType;
+import org.apache.nifi.remote.util.StandardDataPacket;
+import org.apache.nifi.reporting.Severity;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class SocketClientTransaction implements Transaction {
+
+ private static final Logger logger = LoggerFactory.getLogger(SocketClientTransaction.class);
+
+ private final long creationNanoTime = System.nanoTime();
+ private final CRC32 crc = new CRC32();
+ private final int protocolVersion;
+ private final FlowFileCodec codec;
+ private final DataInputStream dis;
+ private final DataOutputStream dos;
+ private final TransferDirection direction;
+ private final boolean compress;
+ private final Peer peer;
+ private final int penaltyMillis;
+ private final String destinationId;
+ private final EventReporter eventReporter;
+
+ private boolean dataAvailable = false;
+ private int transfers = 0;
+ private long contentBytes = 0;
+ private TransactionState state;
+
+ SocketClientTransaction(final int protocolVersion, final String destinationId, final Peer peer, final FlowFileCodec codec,
+ final TransferDirection direction, final boolean useCompression, final int penaltyMillis, final EventReporter eventReporter) throws IOException {
+ this.protocolVersion = protocolVersion;
+ this.destinationId = destinationId;
+ this.peer = peer;
+ this.codec = codec;
+ this.direction = direction;
+ this.dis = new DataInputStream(peer.getCommunicationsSession().getInput().getInputStream());
+ this.dos = new DataOutputStream(peer.getCommunicationsSession().getOutput().getOutputStream());
+ this.compress = useCompression;
+ this.state = TransactionState.TRANSACTION_STARTED;
+ this.penaltyMillis = penaltyMillis;
+ this.eventReporter = eventReporter;
+
+ initialize();
+ }
+
+ private void initialize() throws IOException {
+ try {
+ if (direction == TransferDirection.RECEIVE) {
+ // Indicate that we would like to have some data
+ RequestType.RECEIVE_FLOWFILES.writeRequestType(dos);
+ dos.flush();
+
+ final Response dataAvailableCode = Response.read(dis);
+ switch (dataAvailableCode.getCode()) {
+ case MORE_DATA:
+ logger.debug("{} {} Indicates that data is available", this, peer);
+ this.dataAvailable = true;
+ break;
+ case NO_MORE_DATA:
+ logger.debug("{} No data available from {}", peer);
+ this.dataAvailable = false;
+ return;
+ default:
+ throw new ProtocolException("Got unexpected response when asking for data: " + dataAvailableCode);
+ }
+
+ } else {
+ // Indicate that we would like to have some data
+ RequestType.SEND_FLOWFILES.writeRequestType(dos);
+ dos.flush();
+ }
+ } catch (final Exception e) {
+ error();
+ throw e;
+ }
+ }
+
+ @Override
+ public DataPacket receive() throws IOException {
+ try {
+ try {
+ if (state != TransactionState.DATA_EXCHANGED && state != TransactionState.TRANSACTION_STARTED) {
+ throw new IllegalStateException("Cannot receive data from " + peer + " because Transaction State is " + state);
+ }
+
+ if (direction == TransferDirection.SEND) {
+ throw new IllegalStateException("Attempting to receive data from " + peer + " but started a SEND Transaction");
+ }
+
+ // if we already know there's no data, just return null
+ if (!dataAvailable) {
+ return null;
+ }
+
+ // if we have already received a packet, check if another is available.
+ if (transfers > 0) {
+ // Determine if Peer will send us data or has no data to send us
+ final Response dataAvailableCode = Response.read(dis);
+ switch (dataAvailableCode.getCode()) {
+ case CONTINUE_TRANSACTION:
+ logger.debug("{} {} Indicates Transaction should continue", this, peer);
+ this.dataAvailable = true;
+ break;
+ case FINISH_TRANSACTION:
+ logger.debug("{} {} Indicates Transaction should finish", this, peer);
+ this.dataAvailable = false;
+ break;
+ default:
+ throw new ProtocolException("Got unexpected response from " + peer + " when asking for data: " + dataAvailableCode);
+ }
+ }
+
+ // if no data available, return null
+ if (!dataAvailable) {
+ return null;
+ }
+
+ logger.debug("{} Receiving data from {}", this, peer);
+ final InputStream dataIn = compress ? new CompressionInputStream(dis) : dis;
+ final DataPacket packet = codec.decode(new CheckedInputStream(dataIn, crc));
+
+ if (packet == null) {
+ this.dataAvailable = false;
+ } else {
+ transfers++;
+ contentBytes += packet.getSize();
+ }
+
+ this.state = TransactionState.DATA_EXCHANGED;
+ return packet;
+ } catch (final IOException ioe) {
+ throw new IOException("Failed to receive data from " + peer + " due to " + ioe, ioe);
+ }
+ } catch (final Exception e) {
+ error();
+ throw e;
+ }
+ }
+
+ @Override
+ public void send(final byte[] content, final Map<String, String> attributes) throws IOException {
+ send(new StandardDataPacket(attributes, new ByteArrayInputStream(content), content.length));
+ }
+
+ @Override
+ public void send(final DataPacket dataPacket) throws IOException {
+ try {
+ try {
+ if (state != TransactionState.DATA_EXCHANGED && state != TransactionState.TRANSACTION_STARTED) {
+ throw new IllegalStateException("Cannot send data to " + peer + " because Transaction State is " + state);
+ }
+
+ if (direction == TransferDirection.RECEIVE) {
+ throw new IllegalStateException("Attempting to send data to " + peer + " but started a RECEIVE Transaction");
+ }
+
+ if (transfers > 0) {
+ ResponseCode.CONTINUE_TRANSACTION.writeResponse(dos);
+ }
+
+ logger.debug("{} Sending data to {}", this, peer);
+
+ final OutputStream dataOut = compress ? new CompressionOutputStream(dos) : dos;
+ final OutputStream out = new CheckedOutputStream(dataOut, crc);
+ codec.encode(dataPacket, out);
+
+ // need to close the CompressionOutputStream in order to force it write out any remaining bytes.
+ // Otherwise, do NOT close it because we don't want to close the underlying stream
+ // (CompressionOutputStream will not close the underlying stream when it's closed)
+ if (compress) {
+ out.close();
+ }
+
+ transfers++;
+ contentBytes += dataPacket.getSize();
+ this.state = TransactionState.DATA_EXCHANGED;
+ } catch (final IOException ioe) {
+ throw new IOException("Failed to send data to " + peer + " due to " + ioe, ioe);
+ }
+ } catch (final Exception e) {
+ error();
+ throw e;
+ }
+ }
+
+ @Override
+ public void cancel(final String explanation) throws IOException {
+ if (state == TransactionState.TRANSACTION_CANCELED || state == TransactionState.TRANSACTION_COMPLETED || state == TransactionState.ERROR) {
+ throw new IllegalStateException("Cannot cancel transaction because state is already " + state);
+ }
+
+ try {
+ ResponseCode.CANCEL_TRANSACTION.writeResponse(dos, explanation == null ? "<No explanation given>" : explanation);
+ state = TransactionState.TRANSACTION_CANCELED;
+ } catch (final IOException ioe) {
+ error();
+ throw new IOException("Failed to send 'cancel transaction' message to " + peer + " due to " + ioe, ioe);
+ }
+ }
+
+ @Override
+ public TransactionCompletion complete() throws IOException {
+ try {
+ try {
+ if (state != TransactionState.TRANSACTION_CONFIRMED) {
+ throw new IllegalStateException("Cannot complete transaction with " + peer + " because state is " + state
+ + "; Transaction can only be completed when state is " + TransactionState.TRANSACTION_CONFIRMED);
+ }
+
+ boolean backoff = false;
+ if (direction == TransferDirection.RECEIVE) {
+ if (transfers == 0) {
+ state = TransactionState.TRANSACTION_COMPLETED;
+ return new SocketClientTransactionCompletion(false, 0, 0L, System.nanoTime() - creationNanoTime);
+ }
+
+ // Confirm that we received the data and the peer can now discard it
+ logger.debug("{} Sending TRANSACTION_FINISHED to {}", this, peer);
+ ResponseCode.TRANSACTION_FINISHED.writeResponse(dos);
+
+ state = TransactionState.TRANSACTION_COMPLETED;
+ } else {
+ final Response transactionResponse;
+ try {
+ transactionResponse = Response.read(dis);
+ } catch (final IOException e) {
+ throw new IOException(this + " Failed to receive a response from " + peer + " when expecting a TransactionFinished Indicator. "
+ + "It is unknown whether or not the peer successfully received/processed the data.", e);
+ }
+
+ logger.debug("{} Received {} from {}", this, transactionResponse, peer);
+ if (transactionResponse.getCode() == ResponseCode.TRANSACTION_FINISHED_BUT_DESTINATION_FULL) {
+ peer.penalize(destinationId, penaltyMillis);
+ backoff = true;
+ } else if (transactionResponse.getCode() != ResponseCode.TRANSACTION_FINISHED) {
+ throw new ProtocolException("After sending data to " + peer + ", expected TRANSACTION_FINISHED response but got " + transactionResponse);
+ }
+
+ state = TransactionState.TRANSACTION_COMPLETED;
+ }
+
+ return new SocketClientTransactionCompletion(backoff, transfers, contentBytes, System.nanoTime() - creationNanoTime);
+ } catch (final IOException ioe) {
+ throw new IOException("Failed to complete transaction with " + peer + " due to " + ioe, ioe);
+ }
+ } catch (final Exception e) {
+ error();
+ throw e;
+ }
+ }
+
+ @Override
+ public void confirm() throws IOException {
+ try {
+ try {
+ if (state == TransactionState.TRANSACTION_STARTED && !dataAvailable && direction == TransferDirection.RECEIVE) {
+ // client requested to receive data but no data available. no need to confirm.
+ state = TransactionState.TRANSACTION_CONFIRMED;
+ return;
+ }
+
+ if (state != TransactionState.DATA_EXCHANGED) {
+ throw new IllegalStateException("Cannot confirm Transaction because state is " + state
+ + "; Transaction can only be confirmed when state is " + TransactionState.DATA_EXCHANGED);
+ }
+
+ if (direction == TransferDirection.RECEIVE) {
+ if (dataAvailable) {
+ throw new IllegalStateException("Cannot complete transaction because the sender has already sent more data than client has consumed.");
+ }
+
+ // we received a FINISH_TRANSACTION indicator. Send back a CONFIRM_TRANSACTION message
+ // to peer so that we can verify that the connection is still open. This is a two-phase commit,
+ // which helps to prevent the chances of data duplication. Without doing this, we may commit the
+ // session and then when we send the response back to the peer, the peer may have timed out and may not
+ // be listening. As a result, it will re-send the data. By doing this two-phase commit, we narrow the
+ // Critical Section involved in this transaction so that rather than the Critical Section being the
+ // time window involved in the entire transaction, it is reduced to a simple round-trip conversation.
+ logger.trace("{} Sending CONFIRM_TRANSACTION Response Code to {}", this, peer);
+ final String calculatedCRC = String.valueOf(crc.getValue());
+ ResponseCode.CONFIRM_TRANSACTION.writeResponse(dos, calculatedCRC);
+
+ final Response confirmTransactionResponse;
+ try {
+ confirmTransactionResponse = Response.read(dis);
+ } catch (final IOException ioe) {
+ logger.error("Failed to receive response code from {} when expecting confirmation of transaction", peer);
+ if (eventReporter != null) {
+ eventReporter.reportEvent(Severity.ERROR, "Site-to-Site", "Failed to receive response code from " + peer + " when expecting confirmation of transaction");
+ }
+ throw ioe;
+ }
+
+ logger.trace("{} Received {} from {}", this, confirmTransactionResponse, peer);
+
+ switch (confirmTransactionResponse.getCode()) {
+ case CONFIRM_TRANSACTION:
+ break;
+ case BAD_CHECKSUM:
+ throw new IOException(this + " Received a BadChecksum response from peer " + peer);
+ default:
+ throw new ProtocolException(this + " Received unexpected Response from peer " + peer + " : "
+ + confirmTransactionResponse + "; expected 'Confirm Transaction' Response Code");
+ }
+
+ state = TransactionState.TRANSACTION_CONFIRMED;
+ } else {
+ logger.debug("{} Sent FINISH_TRANSACTION indicator to {}", this, peer);
+ ResponseCode.FINISH_TRANSACTION.writeResponse(dos);
+
+ final String calculatedCRC = String.valueOf(crc.getValue());
+
+ // we've sent a FINISH_TRANSACTION. Now we'll wait for the peer to send a 'Confirm Transaction' response
+ final Response transactionConfirmationResponse = Response.read(dis);
+ if (transactionConfirmationResponse.getCode() == ResponseCode.CONFIRM_TRANSACTION) {
+ // Confirm checksum and echo back the confirmation.
+ logger.trace("{} Received {} from {}", this, transactionConfirmationResponse, peer);
+ final String receivedCRC = transactionConfirmationResponse.getMessage();
+
+ // CRC was not used before version 4
+ if (protocolVersion > 3) {
+ if (!receivedCRC.equals(calculatedCRC)) {
+ ResponseCode.BAD_CHECKSUM.writeResponse(dos);
+ throw new IOException(this + " Sent data to peer " + peer + " but calculated CRC32 Checksum as "
+ + calculatedCRC + " while peer calculated CRC32 Checksum as "
+ + receivedCRC + "; canceling transaction and rolling back session");
+ }
+ }
+
+ ResponseCode.CONFIRM_TRANSACTION.writeResponse(dos, "");
+ } else {
+ throw new ProtocolException("Expected to receive 'Confirm Transaction' response from peer "
+ + peer + " but received " + transactionConfirmationResponse);
+ }
+
+ state = TransactionState.TRANSACTION_CONFIRMED;
+ }
+ } catch (final IOException ioe) {
+ throw new IOException("Failed to confirm transaction with " + peer + " due to " + ioe, ioe);
+ }
+ } catch (final Exception e) {
+ error();
+ throw e;
+ }
+ }
+
+ @Override
+ public void error() {
+ this.state = TransactionState.ERROR;
+ }
+
+ @Override
+ public TransactionState getState() {
+ return state;
+ }
+
+ @Override
+ public Communicant getCommunicant() {
+ return peer;
+ }
+
+ @Override
+ public String toString() {
+ return "SocketClientTransaction[Url=" + peer.getUrl() + ", TransferDirection=" + direction + ", State=" + state + "]";
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/socket/SocketClientTransactionCompletion.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/socket/SocketClientTransactionCompletion.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/socket/SocketClientTransactionCompletion.java
new file mode 100644
index 0000000..bd95013
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/protocol/socket/SocketClientTransactionCompletion.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.protocol.socket;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.nifi.remote.TransactionCompletion;
+
+public class SocketClientTransactionCompletion implements TransactionCompletion {
+
+ private final boolean backoff;
+ private final int dataPacketsTransferred;
+ private final long bytesTransferred;
+ private final long durationNanos;
+
+ public SocketClientTransactionCompletion(final boolean backoff, final int dataPacketsTransferred, final long bytesTransferred, final long durationNanos) {
+ this.backoff = backoff;
+ this.dataPacketsTransferred = dataPacketsTransferred;
+ this.bytesTransferred = bytesTransferred;
+ this.durationNanos = durationNanos;
+ }
+
+ @Override
+ public boolean isBackoff() {
+ return backoff;
+ }
+
+ @Override
+ public int getDataPacketsTransferred() {
+ return dataPacketsTransferred;
+ }
+
+ @Override
+ public long getBytesTransferred() {
+ return bytesTransferred;
+ }
+
+ @Override
+ public long getDuration(final TimeUnit timeUnit) {
+ return timeUnit.convert(durationNanos, TimeUnit.NANOSECONDS);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/util/NiFiRestApiUtil.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/util/NiFiRestApiUtil.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/util/NiFiRestApiUtil.java
new file mode 100644
index 0000000..d746abf
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/util/NiFiRestApiUtil.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.util;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLSession;
+
+import org.apache.nifi.stream.io.StreamUtils;
+import org.apache.nifi.web.api.dto.ControllerDTO;
+import org.codehaus.jackson.JsonNode;
+import org.codehaus.jackson.map.ObjectMapper;
+
+public class NiFiRestApiUtil {
+
+ public static final int RESPONSE_CODE_OK = 200;
+
+ private final SSLContext sslContext;
+
+ public NiFiRestApiUtil(final SSLContext sslContext) {
+ this.sslContext = sslContext;
+ }
+
+ private HttpURLConnection getConnection(final String connUrl, final int timeoutMillis) throws IOException {
+ final URL url = new URL(connUrl);
+ final HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setConnectTimeout(timeoutMillis);
+ connection.setReadTimeout(timeoutMillis);
+
+ // special handling for https
+ if (sslContext != null && connection instanceof HttpsURLConnection) {
+ HttpsURLConnection secureConnection = (HttpsURLConnection) connection;
+ secureConnection.setSSLSocketFactory(sslContext.getSocketFactory());
+
+ // check the trusted hostname property and override the HostnameVerifier
+ secureConnection.setHostnameVerifier(new OverrideHostnameVerifier(url.getHost(),
+ secureConnection.getHostnameVerifier()));
+ }
+
+ return connection;
+ }
+
+ public ControllerDTO getController(final String url, final int timeoutMillis) throws IOException {
+ final HttpURLConnection connection = getConnection(url, timeoutMillis);
+ connection.setRequestMethod("GET");
+ final int responseCode = connection.getResponseCode();
+
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ StreamUtils.copy(connection.getInputStream(), baos);
+ final String responseMessage = baos.toString();
+
+ if (responseCode == RESPONSE_CODE_OK) {
+ final ObjectMapper mapper = new ObjectMapper();
+ final JsonNode jsonNode = mapper.readTree(responseMessage);
+ final JsonNode controllerNode = jsonNode.get("controller");
+ return mapper.readValue(controllerNode, ControllerDTO.class);
+ } else {
+ throw new IOException("Got HTTP response Code " + responseCode + ": " + connection.getResponseMessage() + " with explanation: " + responseMessage);
+ }
+ }
+
+ private static class OverrideHostnameVerifier implements HostnameVerifier {
+
+ private final String trustedHostname;
+ private final HostnameVerifier delegate;
+
+ private OverrideHostnameVerifier(String trustedHostname, HostnameVerifier delegate) {
+ this.trustedHostname = trustedHostname;
+ this.delegate = delegate;
+ }
+
+ @Override
+ public boolean verify(String hostname, SSLSession session) {
+ if (trustedHostname.equalsIgnoreCase(hostname)) {
+ return true;
+ }
+ return delegate.verify(hostname, session);
+ }
+ }
+}
[24/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/BufferStateManager.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/BufferStateManager.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/BufferStateManager.java
new file mode 100644
index 0000000..6e54d62
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/BufferStateManager.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.io.socket;
+
+import java.nio.ByteBuffer;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class BufferStateManager {
+
+ private static final Logger logger = LoggerFactory.getLogger(BufferStateManager.class);
+
+ private ByteBuffer buffer;
+ private Direction direction = Direction.WRITE;
+
+ public BufferStateManager(final ByteBuffer buffer) {
+ this.buffer = buffer;
+ }
+
+ public BufferStateManager(final ByteBuffer buffer, final Direction direction) {
+ this.buffer = buffer;
+ this.direction = direction;
+ }
+
+ /**
+ * Ensures that the buffer is at least as big as the size specified, resizing the buffer if necessary. This operation MAY change the direction of the buffer.
+ *
+ * @param requiredSize the desired size of the buffer
+ */
+ public void ensureSize(final int requiredSize) {
+ if (buffer.capacity() < requiredSize) {
+ final ByteBuffer newBuffer = ByteBuffer.allocate(requiredSize);
+
+ // we have to read buffer so make sure the direction is correct.
+ if (direction == Direction.WRITE) {
+ buffer.flip();
+ }
+
+ // Copy from buffer to newBuffer
+ newBuffer.put(buffer);
+
+ // Swap the buffers
+ buffer = newBuffer;
+
+ // the new buffer is ready to be written to
+ direction = Direction.WRITE;
+ }
+ }
+
+ public ByteBuffer prepareForWrite(final int requiredSize) {
+ ensureSize(requiredSize);
+
+ if (direction == Direction.READ) {
+ direction = Direction.WRITE;
+ buffer.position(buffer.limit());
+ }
+
+ buffer.limit(buffer.capacity());
+ return buffer;
+ }
+
+ public ByteBuffer prepareForRead(final int requiredSize) {
+ ensureSize(requiredSize);
+
+ if (direction == Direction.WRITE) {
+ direction = Direction.READ;
+ buffer.flip();
+ }
+
+ return buffer;
+ }
+
+ /**
+ * Clears the contents of the buffer and sets direction to WRITE
+ */
+ public void clear() {
+ logger.debug("Clearing {}", buffer);
+ buffer.clear();
+ direction = Direction.WRITE;
+ }
+
+ public void compact() {
+ final String before = buffer.toString();
+ buffer.compact();
+ logger.debug("Before compact: {}, after: {}", before, buffer);
+ direction = Direction.WRITE;
+ }
+
+ public static enum Direction {
+
+ READ, WRITE;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelInputStream.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelInputStream.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelInputStream.java
new file mode 100644
index 0000000..1ec229d
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelInputStream.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.io.socket;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.SocketTimeoutException;
+import java.nio.ByteBuffer;
+import java.nio.channels.ClosedByInterruptException;
+import java.nio.channels.SocketChannel;
+import java.util.concurrent.TimeUnit;
+
+public class SocketChannelInputStream extends InputStream {
+
+ private static final long CHANNEL_EMPTY_WAIT_NANOS = TimeUnit.NANOSECONDS.convert(10, TimeUnit.MILLISECONDS);
+ private final SocketChannel channel;
+ private volatile int timeoutMillis = 30000;
+
+ private final ByteBuffer oneByteBuffer = ByteBuffer.allocate(1);
+ private Byte bufferedByte = null;
+
+ public SocketChannelInputStream(final SocketChannel socketChannel) throws IOException {
+ // this class expects a non-blocking channel
+ socketChannel.configureBlocking(false);
+ this.channel = socketChannel;
+ }
+
+ public void setTimeout(final int timeoutMillis) {
+ this.timeoutMillis = timeoutMillis;
+ }
+
+ public void consume() throws IOException {
+ channel.shutdownInput();
+
+ final byte[] b = new byte[4096];
+ final ByteBuffer buffer = ByteBuffer.wrap(b);
+ int bytesRead;
+ do {
+ bytesRead = channel.read(buffer);
+ buffer.flip();
+ } while (bytesRead > 0);
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (bufferedByte != null) {
+ final int retVal = bufferedByte & 0xFF;
+ bufferedByte = null;
+ return retVal;
+ }
+
+ oneByteBuffer.flip();
+ oneByteBuffer.clear();
+
+ final long maxTime = System.currentTimeMillis() + timeoutMillis;
+ int bytesRead;
+ do {
+ bytesRead = channel.read(oneByteBuffer);
+ if (bytesRead == 0) {
+ if (System.currentTimeMillis() > maxTime) {
+ throw new SocketTimeoutException("Timed out reading from socket");
+ }
+ try {
+ TimeUnit.NANOSECONDS.sleep(CHANNEL_EMPTY_WAIT_NANOS);
+ } catch (InterruptedException e) {
+ close();
+ Thread.currentThread().interrupt(); // set the interrupt status
+ throw new ClosedByInterruptException(); // simulate an interrupted blocked read operation
+ }
+ }
+ } while (bytesRead == 0);
+
+ if (bytesRead == -1) {
+ return -1;
+ }
+ oneByteBuffer.flip();
+ return oneByteBuffer.get() & 0xFF;
+ }
+
+ @Override
+ public int read(final byte[] b) throws IOException {
+ return read(b, 0, b.length);
+ }
+
+ @Override
+ public int read(final byte[] b, final int off, final int len) throws IOException {
+ if (bufferedByte != null) {
+ final byte retVal = bufferedByte;
+ bufferedByte = null;
+ b[off] = retVal;
+ return 1;
+ }
+
+ final ByteBuffer buffer = ByteBuffer.wrap(b, off, len);
+
+ final long maxTime = System.currentTimeMillis() + timeoutMillis;
+ int bytesRead;
+ do {
+ bytesRead = channel.read(buffer);
+ if (bytesRead == 0) {
+ if (System.currentTimeMillis() > maxTime) {
+ throw new SocketTimeoutException("Timed out reading from socket");
+ }
+ try {
+ TimeUnit.NANOSECONDS.sleep(CHANNEL_EMPTY_WAIT_NANOS);
+ } catch (InterruptedException e) {
+ close();
+ Thread.currentThread().interrupt(); // set the interrupt status
+ throw new ClosedByInterruptException(); // simulate an interrupted blocked read operation
+ }
+ }
+ } while (bytesRead == 0);
+
+ return bytesRead;
+ }
+
+ @Override
+ public int available() throws IOException {
+ if (bufferedByte != null) {
+ return 1;
+ }
+
+ isDataAvailable(); // attempt to read from socket
+ return (bufferedByte == null) ? 0 : 1;
+ }
+
+ public boolean isDataAvailable() throws IOException {
+ if (bufferedByte != null) {
+ return true;
+ }
+
+ oneByteBuffer.flip();
+ oneByteBuffer.clear();
+ final int bytesRead = channel.read(oneByteBuffer);
+ if (bytesRead == -1) {
+ throw new EOFException("Peer has closed the stream");
+ }
+ if (bytesRead > 0) {
+ oneByteBuffer.flip();
+ bufferedByte = oneByteBuffer.get();
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Closes the underlying socket channel.
+ *
+ * @throws java.io.IOException for issues closing underlying stream
+ */
+ @Override
+ public void close() throws IOException {
+ channel.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelOutputStream.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelOutputStream.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelOutputStream.java
new file mode 100644
index 0000000..a56d9dd
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelOutputStream.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.io.socket;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.SocketTimeoutException;
+import java.nio.ByteBuffer;
+import java.nio.channels.ClosedByInterruptException;
+import java.nio.channels.SocketChannel;
+import java.util.concurrent.TimeUnit;
+
+public class SocketChannelOutputStream extends OutputStream {
+
+ private static final long CHANNEL_FULL_WAIT_NANOS = TimeUnit.NANOSECONDS.convert(10, TimeUnit.MILLISECONDS);
+ private final SocketChannel channel;
+ private volatile int timeout = 30000;
+
+ private final ByteBuffer oneByteBuffer = ByteBuffer.allocate(1);
+
+ public SocketChannelOutputStream(final SocketChannel socketChannel) throws IOException {
+ // this class expects a non-blocking channel
+ socketChannel.configureBlocking(false);
+ this.channel = socketChannel;
+ }
+
+ public void setTimeout(final int timeoutMillis) {
+ this.timeout = timeoutMillis;
+ }
+
+ @Override
+ public void write(final int b) throws IOException {
+ oneByteBuffer.flip();
+ oneByteBuffer.clear();
+ oneByteBuffer.put((byte) b);
+ oneByteBuffer.flip();
+
+ final int timeoutMillis = this.timeout;
+ long maxTime = System.currentTimeMillis() + timeoutMillis;
+ int bytesWritten;
+ while (oneByteBuffer.hasRemaining()) {
+ bytesWritten = channel.write(oneByteBuffer);
+ if (bytesWritten == 0) {
+ if (System.currentTimeMillis() > maxTime) {
+ throw new SocketTimeoutException("Timed out writing to socket");
+ }
+ try {
+ TimeUnit.NANOSECONDS.sleep(CHANNEL_FULL_WAIT_NANOS);
+ } catch (InterruptedException e) {
+ close();
+ Thread.currentThread().interrupt(); // set the interrupt status
+ throw new ClosedByInterruptException(); // simulate an interrupted blocked write operation
+ }
+ } else {
+ return;
+ }
+ }
+ }
+
+ @Override
+ public void write(final byte[] b) throws IOException {
+ write(b, 0, b.length);
+ }
+
+ @Override
+ public void write(final byte[] b, final int off, final int len) throws IOException {
+ final ByteBuffer buffer = ByteBuffer.wrap(b, off, len);
+
+ final int timeoutMillis = this.timeout;
+ long maxTime = System.currentTimeMillis() + timeoutMillis;
+ int bytesWritten;
+ while (buffer.hasRemaining()) {
+ bytesWritten = channel.write(buffer);
+ if (bytesWritten == 0) {
+ if (System.currentTimeMillis() > maxTime) {
+ throw new SocketTimeoutException("Timed out writing to socket");
+ }
+ try {
+ TimeUnit.NANOSECONDS.sleep(CHANNEL_FULL_WAIT_NANOS);
+ } catch (InterruptedException e) {
+ close();
+ Thread.currentThread().interrupt(); // set the interrupt status
+ throw new ClosedByInterruptException(); // simulate an interrupted blocked write operation
+ }
+ } else {
+ maxTime = System.currentTimeMillis() + timeoutMillis;
+ }
+ }
+ }
+
+ /**
+ * Closes the underlying SocketChannel
+ *
+ * @throws java.io.IOException if issues closing underlying stream
+ */
+ @Override
+ public void close() throws IOException {
+ channel.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannel.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannel.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannel.java
new file mode 100644
index 0000000..1f23d79
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannel.java
@@ -0,0 +1,614 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.io.socket.ssl;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.net.SocketAddress;
+import java.net.SocketTimeoutException;
+import java.nio.ByteBuffer;
+import java.nio.channels.ClosedByInterruptException;
+import java.nio.channels.SocketChannel;
+import java.util.concurrent.TimeUnit;
+
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLEngine;
+import javax.net.ssl.SSLEngineResult;
+import javax.net.ssl.SSLEngineResult.Status;
+import javax.net.ssl.SSLHandshakeException;
+import javax.net.ssl.SSLPeerUnverifiedException;
+import javax.security.cert.CertificateExpiredException;
+import javax.security.cert.CertificateNotYetValidException;
+import javax.security.cert.X509Certificate;
+
+import org.apache.nifi.remote.exception.TransmissionDisabledException;
+import org.apache.nifi.remote.io.socket.BufferStateManager;
+import org.apache.nifi.remote.io.socket.BufferStateManager.Direction;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class SSLSocketChannel implements Closeable {
+
+ public static final int MAX_WRITE_SIZE = 65536;
+
+ private static final Logger logger = LoggerFactory.getLogger(SSLSocketChannel.class);
+ private static final long BUFFER_FULL_EMPTY_WAIT_NANOS = TimeUnit.NANOSECONDS.convert(10, TimeUnit.MILLISECONDS);
+
+ private final String hostname;
+ private final int port;
+ private final SSLEngine engine;
+ private final SocketAddress socketAddress;
+
+ private BufferStateManager streamInManager;
+ private BufferStateManager streamOutManager;
+ private BufferStateManager appDataManager;
+
+ private SocketChannel channel;
+
+ private final byte[] oneByteBuffer = new byte[1];
+
+ private int timeoutMillis = 30000;
+ private volatile boolean connected = false;
+ private boolean handshaking = false;
+ private boolean closed = false;
+ private volatile boolean interrupted = false;
+
+ public SSLSocketChannel(final SSLContext sslContext, final String hostname, final int port, final boolean client) throws IOException {
+ this.socketAddress = new InetSocketAddress(hostname, port);
+ this.channel = SocketChannel.open();
+ this.hostname = hostname;
+ this.port = port;
+ this.engine = sslContext.createSSLEngine();
+ this.engine.setUseClientMode(client);
+ engine.setNeedClientAuth(true);
+
+ streamInManager = new BufferStateManager(ByteBuffer.allocate(engine.getSession().getPacketBufferSize()));
+ streamOutManager = new BufferStateManager(ByteBuffer.allocate(engine.getSession().getPacketBufferSize()));
+ appDataManager = new BufferStateManager(ByteBuffer.allocate(engine.getSession().getApplicationBufferSize()));
+ }
+
+ public SSLSocketChannel(final SSLContext sslContext, final SocketChannel socketChannel, final boolean client) throws IOException {
+ if (!socketChannel.isConnected()) {
+ throw new IllegalArgumentException("Cannot pass an un-connected SocketChannel");
+ }
+
+ this.channel = socketChannel;
+
+ this.socketAddress = socketChannel.getRemoteAddress();
+ final Socket socket = socketChannel.socket();
+ this.hostname = socket.getInetAddress().getHostName();
+ this.port = socket.getPort();
+
+ this.engine = sslContext.createSSLEngine();
+ this.engine.setUseClientMode(client);
+ engine.setNeedClientAuth(true);
+
+ streamInManager = new BufferStateManager(ByteBuffer.allocate(engine.getSession().getPacketBufferSize()));
+ streamOutManager = new BufferStateManager(ByteBuffer.allocate(engine.getSession().getPacketBufferSize()));
+ appDataManager = new BufferStateManager(ByteBuffer.allocate(engine.getSession().getApplicationBufferSize()));
+ }
+
+ public void setTimeout(final int millis) {
+ this.timeoutMillis = millis;
+ }
+
+ public int getTimeout() {
+ return timeoutMillis;
+ }
+
+ public void connect() throws SSLHandshakeException, IOException {
+ try {
+ channel.configureBlocking(false);
+ if (!channel.isConnected()) {
+ final long startTime = System.currentTimeMillis();
+
+ if (!channel.connect(socketAddress)) {
+ while (!channel.finishConnect()) {
+ if (interrupted) {
+ throw new TransmissionDisabledException();
+ }
+ if (System.currentTimeMillis() > startTime + timeoutMillis) {
+ throw new SocketTimeoutException("Timed out connecting to " + hostname + ":" + port);
+ }
+
+ try {
+ Thread.sleep(50L);
+ } catch (final InterruptedException e) {
+ }
+ }
+ }
+ }
+ engine.beginHandshake();
+
+ performHandshake();
+ logger.debug("{} Successfully completed SSL handshake", this);
+
+ streamInManager.clear();
+ streamOutManager.clear();
+ appDataManager.clear();
+
+ connected = true;
+ } catch (final Exception e) {
+ logger.error("{} Failed to connect due to {}", this, e);
+ if (logger.isDebugEnabled()) {
+ logger.error("", e);
+ }
+ closeQuietly(channel);
+ engine.closeInbound();
+ engine.closeOutbound();
+ throw e;
+ }
+ }
+
+ public String getDn() throws CertificateExpiredException, CertificateNotYetValidException, SSLPeerUnverifiedException {
+ final X509Certificate[] certs = engine.getSession().getPeerCertificateChain();
+ if (certs == null || certs.length == 0) {
+ throw new SSLPeerUnverifiedException("No certificates found");
+ }
+
+ final X509Certificate cert = certs[0];
+ cert.checkValidity();
+ return cert.getSubjectDN().getName().trim();
+ }
+
+ private void performHandshake() throws IOException {
+ // Generate handshake message
+ final byte[] emptyMessage = new byte[0];
+ handshaking = true;
+ logger.debug("{} Performing Handshake", this);
+
+ try {
+ while (true) {
+ switch (engine.getHandshakeStatus()) {
+ case FINISHED:
+ return;
+ case NEED_WRAP: {
+ final ByteBuffer appDataOut = ByteBuffer.wrap(emptyMessage);
+
+ final ByteBuffer outboundBuffer = streamOutManager.prepareForWrite(engine.getSession().getApplicationBufferSize());
+
+ final SSLEngineResult wrapHelloResult = engine.wrap(appDataOut, outboundBuffer);
+ if (wrapHelloResult.getStatus() == Status.BUFFER_OVERFLOW) {
+ streamOutManager.prepareForWrite(engine.getSession().getApplicationBufferSize());
+ continue;
+ }
+
+ if (wrapHelloResult.getStatus() != Status.OK) {
+ throw new SSLHandshakeException("Could not generate SSL Handshake information: SSLEngineResult: "
+ + wrapHelloResult.toString());
+ }
+
+ logger.trace("{} Handshake response after wrapping: {}", this, wrapHelloResult);
+
+ final ByteBuffer readableStreamOut = streamOutManager.prepareForRead(1);
+ final int bytesToSend = readableStreamOut.remaining();
+ writeFully(readableStreamOut);
+ logger.trace("{} Sent {} bytes of wrapped data for handshake", this, bytesToSend);
+
+ streamOutManager.clear();
+ }
+ continue;
+ case NEED_UNWRAP: {
+ final ByteBuffer readableDataIn = streamInManager.prepareForRead(0);
+ final ByteBuffer appData = appDataManager.prepareForWrite(engine.getSession().getApplicationBufferSize());
+
+ // Read handshake response from other side
+ logger.trace("{} Unwrapping: {} to {}", new Object[]{this, readableDataIn, appData});
+ SSLEngineResult handshakeResponseResult = engine.unwrap(readableDataIn, appData);
+ logger.trace("{} Handshake response after unwrapping: {}", this, handshakeResponseResult);
+
+ if (handshakeResponseResult.getStatus() == Status.BUFFER_UNDERFLOW) {
+ final ByteBuffer writableDataIn = streamInManager.prepareForWrite(engine.getSession().getPacketBufferSize());
+ final int bytesRead = readData(writableDataIn);
+ if (bytesRead > 0) {
+ logger.trace("{} Read {} bytes for handshake", this, bytesRead);
+ }
+
+ if (bytesRead < 0) {
+ throw new SSLHandshakeException("Reached End-of-File marker while performing handshake");
+ }
+ } else if (handshakeResponseResult.getStatus() == Status.CLOSED) {
+ throw new IOException("Channel was closed by peer during handshake");
+ } else {
+ streamInManager.compact();
+ appDataManager.clear();
+ }
+ }
+ break;
+ case NEED_TASK:
+ performTasks();
+ continue;
+ case NOT_HANDSHAKING:
+ return;
+ }
+ }
+ } finally {
+ handshaking = false;
+ }
+ }
+
+ private void performTasks() {
+ Runnable runnable;
+ while ((runnable = engine.getDelegatedTask()) != null) {
+ runnable.run();
+ }
+ }
+
+ private void closeQuietly(final Closeable closeable) {
+ try {
+ closeable.close();
+ } catch (final Exception e) {
+ }
+ }
+
+ public void consume() throws IOException {
+ channel.shutdownInput();
+
+ final byte[] b = new byte[4096];
+ final ByteBuffer buffer = ByteBuffer.wrap(b);
+ int readCount;
+ do {
+ readCount = channel.read(buffer);
+ buffer.flip();
+ } while (readCount > 0);
+ }
+
+ private int readData(final ByteBuffer dest) throws IOException {
+ final long startTime = System.currentTimeMillis();
+
+ while (true) {
+ if (interrupted) {
+ throw new TransmissionDisabledException();
+ }
+
+ if (dest.remaining() == 0) {
+ return 0;
+ }
+
+ final int readCount = channel.read(dest);
+
+ if (readCount == 0) {
+ if (System.currentTimeMillis() > startTime + timeoutMillis) {
+ throw new SocketTimeoutException("Timed out reading from socket connected to " + hostname + ":" + port);
+ }
+ try {
+ TimeUnit.NANOSECONDS.sleep(BUFFER_FULL_EMPTY_WAIT_NANOS);
+ } catch (InterruptedException e) {
+ close();
+ Thread.currentThread().interrupt(); // set the interrupt status
+ throw new ClosedByInterruptException();
+ }
+
+ continue;
+ }
+
+ logger.trace("{} Read {} bytes", this, readCount);
+ return readCount;
+ }
+ }
+
+ private Status encryptAndWriteFully(final BufferStateManager src) throws IOException {
+ SSLEngineResult result = null;
+
+ final ByteBuffer buff = src.prepareForRead(0);
+ final ByteBuffer outBuff = streamOutManager.prepareForWrite(engine.getSession().getApplicationBufferSize());
+
+ logger.trace("{} Encrypting {} bytes", this, buff.remaining());
+ while (buff.remaining() > 0) {
+ result = engine.wrap(buff, outBuff);
+ if (result.getStatus() == Status.OK) {
+ final ByteBuffer readableOutBuff = streamOutManager.prepareForRead(0);
+ writeFully(readableOutBuff);
+ streamOutManager.clear();
+ } else {
+ return result.getStatus();
+ }
+ }
+
+ return result.getStatus();
+ }
+
+ private void writeFully(final ByteBuffer src) throws IOException {
+ long lastByteWrittenTime = System.currentTimeMillis();
+
+ int bytesWritten = 0;
+ while (src.hasRemaining()) {
+ if (interrupted) {
+ throw new TransmissionDisabledException();
+ }
+
+ final int written = channel.write(src);
+ bytesWritten += written;
+ final long now = System.currentTimeMillis();
+ if (written > 0) {
+ lastByteWrittenTime = now;
+ } else {
+ if (now > lastByteWrittenTime + timeoutMillis) {
+ throw new SocketTimeoutException("Timed out writing to socket connected to " + hostname + ":" + port);
+ }
+ try {
+ TimeUnit.NANOSECONDS.sleep(BUFFER_FULL_EMPTY_WAIT_NANOS);
+ } catch (final InterruptedException e) {
+ close();
+ Thread.currentThread().interrupt(); // set the interrupt status
+ throw new ClosedByInterruptException();
+ }
+ }
+ }
+
+ logger.trace("{} Wrote {} bytes", this, bytesWritten);
+ }
+
+ public boolean isClosed() {
+ if (closed) {
+ return true;
+ }
+ // need to detect if peer has sent closure handshake...if so the answer is true
+ final ByteBuffer writableInBuffer = streamInManager.prepareForWrite(engine.getSession().getPacketBufferSize());
+ int readCount = 0;
+ try {
+ readCount = channel.read(writableInBuffer);
+ } catch (IOException e) {
+ logger.error("{} Failed to readData due to {}", new Object[]{this, e});
+ if (logger.isDebugEnabled()) {
+ logger.error("", e);
+ }
+ readCount = -1; // treat the condition same as if End of Stream
+ }
+ if (readCount == 0) {
+ return false;
+ }
+ if (readCount > 0) {
+ logger.trace("{} Read {} bytes", this, readCount);
+
+ final ByteBuffer streamInBuffer = streamInManager.prepareForRead(1);
+ final ByteBuffer appDataBuffer = appDataManager.prepareForWrite(engine.getSession().getApplicationBufferSize());
+ try {
+ SSLEngineResult unwrapResponse = engine.unwrap(streamInBuffer, appDataBuffer);
+ logger.trace("{} When checking if closed, (handshake={}) Unwrap response: {}", new Object[]{this, handshaking, unwrapResponse});
+ if (unwrapResponse.getStatus().equals(Status.CLOSED)) {
+ // Drain the incoming TCP buffer
+ final ByteBuffer discardBuffer = ByteBuffer.allocate(8192);
+ int bytesDiscarded = channel.read(discardBuffer);
+ while (bytesDiscarded > 0) {
+ discardBuffer.clear();
+ bytesDiscarded = channel.read(discardBuffer);
+ }
+ engine.closeInbound();
+ } else {
+ streamInManager.compact();
+ return false;
+ }
+ } catch (IOException e) {
+ logger.error("{} Failed to check if closed due to {}. Closing channel.", new Object[]{this, e});
+ if (logger.isDebugEnabled()) {
+ logger.error("", e);
+ }
+ }
+ }
+ // either readCount is -1, indicating an end of stream, or the peer sent a closure handshake
+ // so go ahead and close down the channel
+ closeQuietly(channel.socket());
+ closeQuietly(channel);
+ closed = true;
+ return true;
+ }
+
+ @Override
+ public void close() throws IOException {
+ logger.debug("{} Closing Connection", this);
+ if (channel == null) {
+ return;
+ }
+
+ if (closed) {
+ return;
+ }
+
+ try {
+ engine.closeOutbound();
+
+ final byte[] emptyMessage = new byte[0];
+
+ final ByteBuffer appDataOut = ByteBuffer.wrap(emptyMessage);
+ final ByteBuffer outboundBuffer = streamOutManager.prepareForWrite(engine.getSession().getApplicationBufferSize());
+ final SSLEngineResult handshakeResult = engine.wrap(appDataOut, outboundBuffer);
+
+ if (handshakeResult.getStatus() != Status.CLOSED) {
+ throw new IOException("Invalid close state - will not send network data");
+ }
+
+ final ByteBuffer readableStreamOut = streamOutManager.prepareForRead(1);
+ writeFully(readableStreamOut);
+ } finally {
+ // Drain the incoming TCP buffer
+ final ByteBuffer discardBuffer = ByteBuffer.allocate(8192);
+ try {
+ int bytesDiscarded = channel.read(discardBuffer);
+ while (bytesDiscarded > 0) {
+ discardBuffer.clear();
+ bytesDiscarded = channel.read(discardBuffer);
+ }
+ } catch (Exception e) {
+ }
+
+ closeQuietly(channel.socket());
+ closeQuietly(channel);
+ closed = true;
+ }
+ }
+
+ private int copyFromAppDataBuffer(final byte[] buffer, final int offset, final int len) {
+ // If any data already exists in the application data buffer, copy it to the buffer.
+ final ByteBuffer appDataBuffer = appDataManager.prepareForRead(1);
+
+ final int appDataRemaining = appDataBuffer.remaining();
+ if (appDataRemaining > 0) {
+ final int bytesToCopy = Math.min(len, appDataBuffer.remaining());
+ appDataBuffer.get(buffer, offset, bytesToCopy);
+
+ final int bytesCopied = appDataRemaining - appDataBuffer.remaining();
+ logger.trace("{} Copied {} ({}) bytes from unencrypted application buffer to user space",
+ new Object[]{this, bytesToCopy, bytesCopied});
+ return bytesCopied;
+ }
+ return 0;
+ }
+
+ public int available() throws IOException {
+ ByteBuffer appDataBuffer = appDataManager.prepareForRead(1);
+ ByteBuffer streamDataBuffer = streamInManager.prepareForRead(1);
+ final int buffered = appDataBuffer.remaining() + streamDataBuffer.remaining();
+ if (buffered > 0) {
+ return buffered;
+ }
+
+ final boolean wasAbleToRead = isDataAvailable();
+ if (!wasAbleToRead) {
+ return 0;
+ }
+
+ appDataBuffer = appDataManager.prepareForRead(1);
+ streamDataBuffer = streamInManager.prepareForRead(1);
+ return appDataBuffer.remaining() + streamDataBuffer.remaining();
+ }
+
+ public boolean isDataAvailable() throws IOException {
+ final ByteBuffer appDataBuffer = appDataManager.prepareForRead(1);
+ final ByteBuffer streamDataBuffer = streamInManager.prepareForRead(1);
+
+ if (appDataBuffer.remaining() > 0 || streamDataBuffer.remaining() > 0) {
+ return true;
+ }
+
+ final ByteBuffer writableBuffer = streamInManager.prepareForWrite(engine.getSession().getPacketBufferSize());
+ final int bytesRead = channel.read(writableBuffer);
+ return (bytesRead > 0);
+ }
+
+ public int read() throws IOException {
+ final int bytesRead = read(oneByteBuffer);
+ if (bytesRead == -1) {
+ return -1;
+ }
+ return oneByteBuffer[0] & 0xFF;
+ }
+
+ public int read(final byte[] buffer) throws IOException {
+ return read(buffer, 0, buffer.length);
+ }
+
+ public int read(final byte[] buffer, final int offset, final int len) throws IOException {
+ logger.debug("{} Reading up to {} bytes of data", this, len);
+
+ if (!connected) {
+ connect();
+ }
+
+ int copied = copyFromAppDataBuffer(buffer, offset, len);
+ if (copied > 0) {
+ return copied;
+ }
+
+ appDataManager.clear();
+
+ while (true) {
+ // prepare buffers and call unwrap
+ final ByteBuffer streamInBuffer = streamInManager.prepareForRead(1);
+ SSLEngineResult unwrapResponse = null;
+ final ByteBuffer appDataBuffer = appDataManager.prepareForWrite(engine.getSession().getApplicationBufferSize());
+ unwrapResponse = engine.unwrap(streamInBuffer, appDataBuffer);
+ logger.trace("{} When reading data, (handshake={}) Unwrap response: {}", new Object[]{this, handshaking, unwrapResponse});
+
+ switch (unwrapResponse.getStatus()) {
+ case BUFFER_OVERFLOW:
+ throw new SSLHandshakeException("Buffer Overflow, which is not allowed to happen from an unwrap");
+ case BUFFER_UNDERFLOW: {
+// appDataManager.prepareForRead(engine.getSession().getApplicationBufferSize());
+
+ final ByteBuffer writableInBuffer = streamInManager.prepareForWrite(engine.getSession().getPacketBufferSize());
+ final int bytesRead = readData(writableInBuffer);
+ if (bytesRead < 0) {
+ return -1;
+ }
+
+ continue;
+ }
+ case CLOSED:
+ throw new IOException("Channel is closed");
+ case OK: {
+ copied = copyFromAppDataBuffer(buffer, offset, len);
+ if (copied == 0) {
+ throw new IOException("Failed to decrypt data");
+ }
+ streamInManager.compact();
+ return copied;
+ }
+ }
+ }
+ }
+
+ public void write(final int data) throws IOException {
+ write(new byte[]{(byte) data}, 0, 1);
+ }
+
+ public void write(final byte[] data) throws IOException {
+ write(data, 0, data.length);
+ }
+
+ public void write(final byte[] data, final int offset, final int len) throws IOException {
+ logger.debug("{} Writing {} bytes of data", this, len);
+
+ if (!connected) {
+ connect();
+ }
+
+ int iterations = len / MAX_WRITE_SIZE;
+ if (len % MAX_WRITE_SIZE > 0) {
+ iterations++;
+ }
+
+ for (int i = 0; i < iterations; i++) {
+ streamOutManager.clear();
+ final int itrOffset = offset + i * MAX_WRITE_SIZE;
+ final int itrLen = Math.min(len - itrOffset, MAX_WRITE_SIZE);
+ final ByteBuffer byteBuffer = ByteBuffer.wrap(data, itrOffset, itrLen);
+
+ final BufferStateManager buffMan = new BufferStateManager(byteBuffer, Direction.READ);
+ final Status status = encryptAndWriteFully(buffMan);
+ switch (status) {
+ case BUFFER_OVERFLOW:
+ streamOutManager.ensureSize(engine.getSession().getPacketBufferSize());
+ appDataManager.ensureSize(engine.getSession().getApplicationBufferSize());
+ continue;
+ case OK:
+ continue;
+ case CLOSED:
+ throw new IOException("Channel is closed");
+ case BUFFER_UNDERFLOW:
+ throw new AssertionError("Got Buffer Underflow but should not have...");
+ }
+ }
+ }
+
+ public void interrupt() {
+ this.interrupted = true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelInputStream.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelInputStream.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelInputStream.java
new file mode 100644
index 0000000..ca6de85
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelInputStream.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.io.socket.ssl;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+public class SSLSocketChannelInputStream extends InputStream {
+
+ private final SSLSocketChannel channel;
+
+ public SSLSocketChannelInputStream(final SSLSocketChannel channel) {
+ this.channel = channel;
+ }
+
+ public void consume() throws IOException {
+ channel.consume();
+ }
+
+ @Override
+ public int read() throws IOException {
+ return channel.read();
+ }
+
+ @Override
+ public int read(final byte[] b) throws IOException {
+ return channel.read(b);
+ }
+
+ @Override
+ public int read(final byte[] b, final int off, final int len) throws IOException {
+ return channel.read(b, off, len);
+ }
+
+ /**
+ * Closes the underlying SSLSocketChannel, which will also close the OutputStream and connection
+ */
+ @Override
+ public void close() throws IOException {
+ channel.close();
+ }
+
+ @Override
+ public int available() throws IOException {
+ return channel.available();
+ }
+
+ public boolean isDataAvailable() throws IOException {
+ return available() > 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelOutputStream.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelOutputStream.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelOutputStream.java
new file mode 100644
index 0000000..262cf54
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/remote/io/socket/ssl/SSLSocketChannelOutputStream.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.io.socket.ssl;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+public class SSLSocketChannelOutputStream extends OutputStream {
+
+ private final SSLSocketChannel channel;
+
+ public SSLSocketChannelOutputStream(final SSLSocketChannel channel) {
+ this.channel = channel;
+ }
+
+ @Override
+ public void write(final int b) throws IOException {
+ channel.write(b);
+ }
+
+ @Override
+ public void write(byte[] b) throws IOException {
+ channel.write(b);
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) throws IOException {
+ channel.write(b, off, len);
+ }
+
+ /**
+ * Closes the underlying SSLSocketChannel, which also will close the InputStream and the connection
+ */
+ @Override
+ public void close() throws IOException {
+ channel.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/BufferedInputStream.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/BufferedInputStream.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/BufferedInputStream.java
new file mode 100644
index 0000000..2afaa70
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/BufferedInputStream.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io;
+
+import java.io.InputStream;
+
+/**
+ * This class is a slight modification of the BufferedInputStream in the java.io package. The modification is that this implementation does not provide synchronization on method calls, which means
+ * that this class is not suitable for use by multiple threads. However, the absence of these synchronized blocks results in potentially much better performance.
+ */
+public class BufferedInputStream extends java.io.BufferedInputStream {
+
+ public BufferedInputStream(final InputStream in) {
+ super(in);
+ }
+
+ public BufferedInputStream(final InputStream in, final int size) {
+ super(in, size);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/BufferedOutputStream.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/BufferedOutputStream.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/BufferedOutputStream.java
new file mode 100644
index 0000000..dc56927
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/BufferedOutputStream.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io;
+
+import java.io.FilterOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * This class is a slight modification of the {@link java.io.BufferedOutputStream} class. This implementation differs in that it does not mark methods as synchronized. This means that this class is
+ * not suitable for writing by multiple concurrent threads. However, the removal of the synchronized keyword results in potentially much better performance.
+ */
+public class BufferedOutputStream extends FilterOutputStream {
+
+ /**
+ * The internal buffer where data is stored.
+ */
+ protected byte buf[];
+
+ /**
+ * The number of valid bytes in the buffer. This value is always in the range <tt>0</tt> through <tt>buf.length</tt>; elements
+ * <tt>buf[0]</tt> through <tt>buf[count-1]</tt> contain valid byte data.
+ */
+ protected int count;
+
+ /**
+ * Creates a new buffered output stream to write data to the specified underlying output stream.
+ *
+ * @param out the underlying output stream.
+ */
+ public BufferedOutputStream(OutputStream out) {
+ this(out, 8192);
+ }
+
+ /**
+ * Creates a new buffered output stream to write data to the specified underlying output stream with the specified buffer size.
+ *
+ * @param out the underlying output stream.
+ * @param size the buffer size.
+ * @exception IllegalArgumentException if size <= 0.
+ */
+ public BufferedOutputStream(OutputStream out, int size) {
+ super(out);
+ if (size <= 0) {
+ throw new IllegalArgumentException("Buffer size <= 0");
+ }
+ buf = new byte[size];
+ }
+
+ /**
+ * Flush the internal buffer
+ */
+ private void flushBuffer() throws IOException {
+ if (count > 0) {
+ out.write(buf, 0, count);
+ count = 0;
+ }
+ }
+
+ /**
+ * Writes the specified byte to this buffered output stream.
+ *
+ * @param b the byte to be written.
+ * @exception IOException if an I/O error occurs.
+ */
+ @Override
+ public void write(int b) throws IOException {
+ if (count >= buf.length) {
+ flushBuffer();
+ }
+ buf[count++] = (byte) b;
+ }
+
+ /**
+ * Writes <code>len</code> bytes from the specified byte array starting at offset <code>off</code> to this buffered output stream.
+ *
+ * <p>
+ * Ordinarily this method stores bytes from the given array into this stream's buffer, flushing the buffer to the underlying output stream as needed. If the requested length is at least as large
+ * as this stream's buffer, however, then this method will flush the buffer and write the bytes directly to the underlying output stream. Thus redundant <code>BufferedOutputStream</code>s will not
+ * copy data unnecessarily.
+ *
+ * @param b the data.
+ * @param off the start offset in the data.
+ * @param len the number of bytes to write.
+ * @exception IOException if an I/O error occurs.
+ */
+ @Override
+ public void write(byte b[], int off, int len) throws IOException {
+ if (len >= buf.length) {
+ /* If the request length exceeds the size of the output buffer,
+ flush the output buffer and then write the data directly.
+ In this way buffered streams will cascade harmlessly. */
+ flushBuffer();
+ out.write(b, off, len);
+ return;
+ }
+ if (len >= buf.length - count) {
+ flushBuffer();
+ }
+ System.arraycopy(b, off, buf, count, len);
+ count += len;
+ }
+
+ /**
+ * Flushes this buffered output stream. This forces any buffered output bytes to be written out to the underlying output stream.
+ *
+ * @exception IOException if an I/O error occurs.
+ * @see java.io.FilterOutputStream#out
+ */
+ @Override
+ public void flush() throws IOException {
+ flushBuffer();
+ out.flush();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteArrayInputStream.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteArrayInputStream.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteArrayInputStream.java
new file mode 100644
index 0000000..85c8c4f
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteArrayInputStream.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io;
+
+import java.io.InputStream;
+
+/**
+ * This class performs the same function as java.io.ByteArrayInputStream but does not mark its methods as synchronized
+ */
+public class ByteArrayInputStream extends InputStream {
+
+ /**
+ * An array of bytes that was provided by the creator of the stream. Elements <code>buf[0]</code> through <code>buf[count-1]</code> are the only bytes that can ever be read from the stream;
+ * element <code>buf[pos]</code> is the next byte to be read.
+ */
+ protected byte buf[];
+
+ /**
+ * The index of the next character to read from the input stream buffer. This value should always be nonnegative and not larger than the value of <code>count</code>. The next byte to be read from
+ * the input stream buffer will be <code>buf[pos]</code>.
+ */
+ protected int pos;
+
+ /**
+ * The currently marked position in the stream. ByteArrayInputStream objects are marked at position zero by default when constructed. They may be marked at another position within the buffer by
+ * the <code>mark()</code> method. The current buffer position is set to this point by the <code>reset()</code> method.
+ * <p>
+ * If no mark has been set, then the value of mark is the offset passed to the constructor (or 0 if the offset was not supplied).
+ *
+ * @since JDK1.1
+ */
+ protected int mark = 0;
+
+ /**
+ * The index one greater than the last valid character in the input stream buffer. This value should always be nonnegative and not larger than the length of <code>buf</code>. It is one greater
+ * than the position of the last byte within <code>buf</code> that can ever be read from the input stream buffer.
+ */
+ protected int count;
+
+ /**
+ * Creates a <code>ByteArrayInputStream</code> so that it uses <code>buf</code> as its buffer array. The buffer array is not copied. The initial value of <code>pos</code> is <code>0</code> and the
+ * initial value of <code>count</code> is the length of <code>buf</code>.
+ *
+ * @param buf the input buffer.
+ */
+ public ByteArrayInputStream(byte buf[]) {
+ this.buf = buf;
+ this.pos = 0;
+ this.count = buf.length;
+ }
+
+ /**
+ * Creates <code>ByteArrayInputStream</code> that uses <code>buf</code> as its buffer array. The initial value of <code>pos</code> is <code>offset</code> and the initial value of
+ * <code>count</code> is the minimum of <code>offset+length</code> and <code>buf.length</code>. The buffer array is not copied. The buffer's mark is set to the specified offset.
+ *
+ * @param buf the input buffer.
+ * @param offset the offset in the buffer of the first byte to read.
+ * @param length the maximum number of bytes to read from the buffer.
+ */
+ public ByteArrayInputStream(byte buf[], int offset, int length) {
+ this.buf = buf;
+ this.pos = offset;
+ this.count = Math.min(offset + length, buf.length);
+ this.mark = offset;
+ }
+
+ /**
+ * Reads the next byte of data from this input stream. The value byte is returned as an <code>int</code> in the range <code>0</code> to <code>255</code>. If no byte is available because the end of
+ * the stream has been reached, the value <code>-1</code> is returned.
+ * <p>
+ * This <code>read</code> method cannot block.
+ *
+ * @return the next byte of data, or <code>-1</code> if the end of the stream has been reached.
+ */
+ @Override
+ public int read() {
+ return (pos < count) ? (buf[pos++] & 0xff) : -1;
+ }
+
+ /**
+ * Reads up to <code>len</code> bytes of data into an array of bytes from this input stream. If <code>pos</code> equals <code>count</code>, then <code>-1</code> is returned to indicate end of
+ * file. Otherwise, the number <code>k</code> of bytes read is equal to the smaller of <code>len</code> and <code>count-pos</code>. If <code>k</code> is positive, then bytes <code>buf[pos]</code>
+ * through <code>buf[pos+k-1]</code> are copied into <code>b[off]</code> through <code>b[off+k-1]</code> in the manner performed by <code>System.arraycopy</code>. The value <code>k</code> is added
+ * into <code>pos</code> and <code>k</code> is returned.
+ * <p>
+ * This <code>read</code> method cannot block.
+ *
+ * @param b the buffer into which the data is read.
+ * @param off the start offset in the destination array <code>b</code>
+ * @param len the maximum number of bytes read.
+ * @return the total number of bytes read into the buffer, or <code>-1</code> if there is no more data because the end of the stream has been reached.
+ * @exception NullPointerException If <code>b</code> is <code>null</code>.
+ * @exception IndexOutOfBoundsException If <code>off</code> is negative, <code>len</code> is negative, or <code>len</code> is greater than <code>b.length - off</code>
+ */
+ @Override
+ public int read(byte b[], int off, int len) {
+ if (b == null) {
+ throw new NullPointerException();
+ } else if (off < 0 || len < 0 || len > b.length - off) {
+ throw new IndexOutOfBoundsException();
+ }
+
+ if (pos >= count) {
+ return -1;
+ }
+
+ int avail = count - pos;
+ if (len > avail) {
+ len = avail;
+ }
+ if (len <= 0) {
+ return 0;
+ }
+ System.arraycopy(buf, pos, b, off, len);
+ pos += len;
+ return len;
+ }
+
+ /**
+ * Skips <code>n</code> bytes of input from this input stream. Fewer bytes might be skipped if the end of the input stream is reached. The actual number <code>k</code> of bytes to be skipped is
+ * equal to the smaller of <code>n</code> and <code>count-pos</code>. The value <code>k</code> is added into <code>pos</code> and <code>k</code> is returned.
+ *
+ * @param n the number of bytes to be skipped.
+ * @return the actual number of bytes skipped.
+ */
+ @Override
+ public long skip(long n) {
+ long k = count - pos;
+ if (n < k) {
+ k = n < 0 ? 0 : n;
+ }
+
+ pos += k;
+ return k;
+ }
+
+ /**
+ * Returns the number of remaining bytes that can be read (or skipped over) from this input stream.
+ * <p>
+ * The value returned is <code>count - pos</code>, which is the number of bytes remaining to be read from the input buffer.
+ *
+ * @return the number of remaining bytes that can be read (or skipped over) from this input stream without blocking.
+ */
+ @Override
+ public int available() {
+ return count - pos;
+ }
+
+ /**
+ * Tests if this <code>InputStream</code> supports mark/reset. The <code>markSupported</code> method of <code>ByteArrayInputStream</code> always returns <code>true</code>.
+ *
+ * @since JDK1.1
+ */
+ @Override
+ public boolean markSupported() {
+ return true;
+ }
+
+ /**
+ * Set the current marked position in the stream. ByteArrayInputStream objects are marked at position zero by default when constructed. They may be marked at another position within the buffer by
+ * this method.
+ * <p>
+ * If no mark has been set, then the value of the mark is the offset passed to the constructor (or 0 if the offset was not supplied).
+ *
+ * <p>
+ * Note: The <code>readAheadLimit</code> for this class has no meaning.
+ *
+ * @since JDK1.1
+ */
+ @Override
+ public void mark(int readAheadLimit) {
+ mark = pos;
+ }
+
+ /**
+ * Resets the buffer to the marked position. The marked position is 0 unless another position was marked or an offset was specified in the constructor.
+ */
+ @Override
+ public void reset() {
+ pos = mark;
+ }
+
+ /**
+ * Closing a <tt>ByteArrayInputStream</tt> has no effect. The methods in this class can be called after the stream has been closed without generating an <tt>IOException</tt>.
+ * <p>
+ */
+ @Override
+ public void close() {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteArrayOutputStream.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteArrayOutputStream.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteArrayOutputStream.java
new file mode 100644
index 0000000..aade199
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteArrayOutputStream.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.UnsupportedEncodingException;
+import java.util.Arrays;
+
+/**
+ * This class provides a more efficient implementation of the java.io.ByteArrayOutputStream. The efficiency is gained in two ways:
+ * <ul>
+ * <li>The write methods are not synchronized</li>
+ * <li>The class provides {@link #getUnderlyingBuffer()} and {@link #getBufferLength()}, which can be used to access the underlying byte array directly, rather than the System.arraycopy that
+ * {@link #toByteArray()} uses
+ * </ul>
+ *
+ */
+public class ByteArrayOutputStream extends OutputStream {
+
+ /**
+ * The buffer where data is stored.
+ */
+ protected byte buf[];
+
+ /**
+ * The number of valid bytes in the buffer.
+ */
+ protected int count;
+
+ /**
+ * Creates a new byte array output stream. The buffer capacity is initially 32 bytes, though its size increases if necessary.
+ */
+ public ByteArrayOutputStream() {
+ this(32);
+ }
+
+ /**
+ * Creates a new byte array output stream, with a buffer capacity of the specified size, in bytes.
+ *
+ * @param size the initial size.
+ * @exception IllegalArgumentException if size is negative.
+ */
+ public ByteArrayOutputStream(int size) {
+ if (size < 0) {
+ throw new IllegalArgumentException("Negative initial size: "
+ + size);
+ }
+ buf = new byte[size];
+ }
+
+ /**
+ * Increases the capacity if necessary to ensure that it can hold at least the number of elements specified by the minimum capacity argument.
+ *
+ * @param minCapacity the desired minimum capacity
+ * @throws OutOfMemoryError if {@code minCapacity < 0}. This is interpreted as a request for the unsatisfiably large capacity {@code (long) Integer.MAX_VALUE + (minCapacity - Integer.MAX_VALUE)}.
+ */
+ private void ensureCapacity(int minCapacity) {
+ // overflow-conscious code
+ if (minCapacity - buf.length > 0) {
+ grow(minCapacity);
+ }
+ }
+
+ /**
+ * Increases the capacity to ensure that it can hold at least the number of elements specified by the minimum capacity argument.
+ *
+ * @param minCapacity the desired minimum capacity
+ */
+ private void grow(int minCapacity) {
+ // overflow-conscious code
+ int oldCapacity = buf.length;
+ int newCapacity = oldCapacity << 1;
+ if (newCapacity - minCapacity < 0) {
+ newCapacity = minCapacity;
+ }
+ if (newCapacity < 0) {
+ if (minCapacity < 0) { // overflow
+ throw new OutOfMemoryError();
+ }
+ newCapacity = Integer.MAX_VALUE;
+ }
+ buf = Arrays.copyOf(buf, newCapacity);
+ }
+
+ /**
+ * Writes the specified byte to this byte array output stream.
+ *
+ * @param b the byte to be written.
+ */
+ @Override
+ public void write(int b) {
+ ensureCapacity(count + 1);
+ buf[count] = (byte) b;
+ count += 1;
+ }
+
+ /**
+ * Writes <code>len</code> bytes from the specified byte array starting at offset <code>off</code> to this byte array output stream.
+ *
+ * @param b the data.
+ * @param off the start offset in the data.
+ * @param len the number of bytes to write.
+ */
+ @Override
+ public void write(byte b[], int off, int len) {
+ if ((off < 0) || (off > b.length) || (len < 0)
+ || ((off + len) - b.length > 0)) {
+ throw new IndexOutOfBoundsException();
+ }
+ ensureCapacity(count + len);
+ System.arraycopy(b, off, buf, count, len);
+ count += len;
+ }
+
+ /**
+ * Writes the complete contents of this byte array output stream to the specified output stream argument, as if by calling the output stream's write method using
+ * <code>out.write(buf, 0, count)</code>.
+ *
+ * @param out the output stream to which to write the data.
+ * @exception IOException if an I/O error occurs.
+ */
+ public void writeTo(OutputStream out) throws IOException {
+ out.write(buf, 0, count);
+ }
+
+ /**
+ * Resets the <code>count</code> field of this byte array output stream to zero, so that all currently accumulated output in the output stream is discarded. The output stream can be used again,
+ * reusing the already allocated buffer space.
+ *
+ * @see java.io.ByteArrayInputStream#count
+ */
+ public void reset() {
+ count = 0;
+ }
+
+ /**
+ * Creates a newly allocated byte array. Its size is the current size of this output stream and the valid contents of the buffer have been copied into it.
+ *
+ * @return the current contents of this output stream, as a byte array.
+ * @see java.io.ByteArrayOutputStream#size()
+ */
+ public byte[] toByteArray() {
+ return Arrays.copyOf(buf, count);
+ }
+
+ /**
+ * Returns the current size of the buffer.
+ *
+ * @return the value of the <code>count</code> field, which is the number of valid bytes in this output stream.
+ * @see java.io.ByteArrayOutputStream#count
+ */
+ public int size() {
+ return count;
+ }
+
+ /**
+ * Converts the buffer's contents into a string decoding bytes using the platform's default character set. The length of the new <tt>String</tt>
+ * is a function of the character set, and hence may not be equal to the size of the buffer.
+ *
+ * <p>
+ * This method always replaces malformed-input and unmappable-character sequences with the default replacement string for the platform's default character set. The
+ * {@linkplain java.nio.charset.CharsetDecoder} class should be used when more control over the decoding process is required.
+ *
+ * @return String decoded from the buffer's contents.
+ * @since JDK1.1
+ */
+ @Override
+ public String toString() {
+ return new String(buf, 0, count);
+ }
+
+ /**
+ * Converts the buffer's contents into a string by decoding the bytes using the specified {@link java.nio.charset.Charset charsetName}. The length of the new <tt>String</tt> is a function of the
+ * charset, and hence may not be equal to the length of the byte array.
+ *
+ * <p>
+ * This method always replaces malformed-input and unmappable-character sequences with this charset's default replacement string. The {@link
+ * java.nio.charset.CharsetDecoder} class should be used when more control over the decoding process is required.
+ *
+ * @param charsetName the name of a supported {@linkplain java.nio.charset.Charset <code>charset</code>}
+ * @return String decoded from the buffer's contents.
+ * @exception UnsupportedEncodingException If the named charset is not supported
+ * @since JDK1.1
+ */
+ public String toString(String charsetName) throws UnsupportedEncodingException {
+ return new String(buf, 0, count, charsetName);
+ }
+
+ /**
+ * Closing a <tt>ByteArrayOutputStream</tt> has no effect. The methods in this class can be called after the stream has been closed without generating an <tt>IOException</tt>.
+ * <p>
+ *
+ */
+ @Override
+ public void close() {
+ }
+
+ public byte[] getUnderlyingBuffer() {
+ return buf;
+ }
+
+ public int getBufferLength() {
+ return count;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteCountingInputStream.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteCountingInputStream.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteCountingInputStream.java
new file mode 100644
index 0000000..e9b8c9e
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteCountingInputStream.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+public class ByteCountingInputStream extends InputStream {
+
+ private final InputStream in;
+ private long bytesRead = 0L;
+ private long bytesSkipped = 0L;
+
+ private long bytesSinceMark = 0L;
+
+ public ByteCountingInputStream(final InputStream in) {
+ this.in = in;
+ }
+
+ public ByteCountingInputStream(final InputStream in, final long initialOffset) {
+ this.in = in;
+ this.bytesSkipped = initialOffset;
+ }
+
+ @Override
+ public int read() throws IOException {
+ final int fromSuper = in.read();
+ if (fromSuper >= 0) {
+ bytesRead++;
+ bytesSinceMark++;
+ }
+ return fromSuper;
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ final int fromSuper = in.read(b, off, len);
+ if (fromSuper >= 0) {
+ bytesRead += fromSuper;
+ bytesSinceMark += fromSuper;
+ }
+
+ return fromSuper;
+ }
+
+ @Override
+ public int read(byte[] b) throws IOException {
+ return read(b, 0, b.length);
+ }
+
+ @Override
+ public long skip(final long n) throws IOException {
+ final long skipped = in.skip(n);
+ if (skipped >= 0) {
+ bytesSkipped += skipped;
+ bytesSinceMark += skipped;
+ }
+ return skipped;
+ }
+
+ public long getBytesRead() {
+ return bytesRead;
+ }
+
+ public long getBytesSkipped() {
+ return bytesSkipped;
+ }
+
+ public long getBytesConsumed() {
+ return getBytesRead() + getBytesSkipped();
+ }
+
+ @Override
+ public void mark(final int readlimit) {
+ in.mark(readlimit);
+
+ bytesSinceMark = 0L;
+ }
+
+ @Override
+ public boolean markSupported() {
+ return in.markSupported();
+ }
+
+ @Override
+ public void reset() throws IOException {
+ in.reset();
+ bytesRead -= bytesSinceMark;
+ bytesSinceMark = 0L;
+ }
+
+ @Override
+ public void close() throws IOException {
+ in.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteCountingOutputStream.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteCountingOutputStream.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteCountingOutputStream.java
new file mode 100644
index 0000000..9bbd45e
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteCountingOutputStream.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+public class ByteCountingOutputStream extends OutputStream {
+
+ private final OutputStream out;
+ private long bytesWritten = 0L;
+
+ public ByteCountingOutputStream(final OutputStream out) {
+ this.out = out;
+ }
+
+ public ByteCountingOutputStream(final OutputStream out, final long initialByteCount) {
+ this.out = out;
+ this.bytesWritten = initialByteCount;
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ out.write(b);
+ bytesWritten++;
+ }
+
+ @Override
+ public void write(byte[] b) throws IOException {
+ write(b, 0, b.length);
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) throws IOException {
+ out.write(b, off, len);
+ bytesWritten += len;
+ }
+
+ public long getBytesWritten() {
+ return bytesWritten;
+ }
+
+ @Override
+ public void flush() throws IOException {
+ out.flush();
+ }
+
+ @Override
+ public void close() throws IOException {
+ out.close();
+ }
+}
[46/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/expression/ExpressionLanguageCompiler.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/expression/ExpressionLanguageCompiler.java b/nifi-api/src/main/java/org/apache/nifi/expression/ExpressionLanguageCompiler.java
new file mode 100644
index 0000000..9383d27
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/expression/ExpressionLanguageCompiler.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.expression;
+
+import org.apache.nifi.expression.AttributeExpression.ResultType;
+
+public interface ExpressionLanguageCompiler {
+
+ /**
+ * Compiles the given Attribute Expression string into an
+ * AttributeExpression that can be evaluated
+ *
+ * @param expression the Attribute Expression to be compiled
+ * @return expression that can be evaluated
+ * @throws IllegalArgumentException if the given expression is not valid
+ */
+ AttributeExpression compile(String expression) throws IllegalArgumentException;
+
+ /**
+ * Indicates whether or not the given string is a valid Attribute
+ * Expression.
+ *
+ * @param expression to validate
+ * @return if is value or not
+ */
+ boolean isValidExpression(String expression);
+
+ /**
+ * Attempts to validate the given expression and returns <code>null</code>
+ * if the expression is syntactically valid or a String indicating why the
+ * expression is invalid otherwise.
+ *
+ * @param expression to validate
+ * @param allowSurroundingCharacters if <code>true</code> allows characters
+ * to surround the Expression, otherwise the expression must be exactly
+ * equal to a valid Expression. E.g., <code>/${path}</code> is valid if and
+ * only if <code>allowSurroundingCharacters</code> is true
+ *
+ * @return a String indicating the reason that the expression is not
+ * syntactically correct, or <code>null</code> if the expression is
+ * syntactically correct
+ */
+ String validateExpression(String expression, boolean allowSurroundingCharacters);
+
+ /**
+ * Returns the ResultType that will be returned by the given Expression
+ *
+ * @param expression the Expression to evaluate
+ * @return result type for the given expression
+ * @throws IllegalArgumentException if the given Expression is not a valid
+ * Expression Language Expression; the message of this Exception will
+ * indicate the problem if the expression is not syntactically valid.
+ */
+ ResultType getResultType(String expression) throws IllegalArgumentException;
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/flowfile/FlowFile.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/flowfile/FlowFile.java b/nifi-api/src/main/java/org/apache/nifi/flowfile/FlowFile.java
new file mode 100644
index 0000000..0e2c19d
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/flowfile/FlowFile.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.flowfile;
+
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * <p>
+ * A flow file is a logical notion of an item in a flow with its associated
+ * attributes and identity which can be used as a reference for its actual
+ * content.</p>
+ *
+ * <b>All FlowFile implementations must be Immutable - Thread safe.</b>
+ */
+public interface FlowFile extends Comparable<FlowFile> {
+
+ /**
+ * @return the unique identifier for this flow file
+ */
+ long getId();
+
+ /**
+ * @return the date at which the flow file entered the flow
+ */
+ long getEntryDate();
+
+ /**
+ * @return the date at which the origin of this FlowFile entered the flow.
+ * For example, if FlowFile Z were derived from FlowFile Y and FlowFile Y
+ * was derived from FlowFile X, this date would be the entryDate (see
+ * {@link #getEntryDate()} of FlowFile X.
+ */
+ long getLineageStartDate();
+
+ /**
+ * @return the time at which the FlowFile was most recently added to a
+ * FlowFile queue, or {@code null} if the FlowFile has never been enqueued.
+ * This value will always be populated before it is passed to a
+ * {@link FlowFilePrioritizer}
+ */
+ Long getLastQueueDate();
+
+ /**
+ * <p>
+ * If a FlowFile is derived from multiple "parent" FlowFiles, all of the
+ * parents' Lineage Identifiers will be in the set.
+ * </p>
+ *
+ * @return a set of identifiers that are unique to this FlowFile's lineage.
+ * If FlowFile X is derived from FlowFile Y, both FlowFiles will have the
+ * same value for the Lineage Claim ID.
+ */
+ Set<String> getLineageIdentifiers();
+
+ /**
+ * @return true if flow file is currently penalized; false otherwise;
+ */
+ boolean isPenalized();
+
+ /**
+ * Obtains the attribute value for the given key
+ *
+ * @param key of the attribute
+ * @return value if found; null otherwise
+ */
+ String getAttribute(String key);
+
+ /**
+ * @return size of flow file contents in bytes
+ */
+ long getSize();
+
+ /**
+ * @return an unmodifiable map of the flow file attributes
+ */
+ Map<String, String> getAttributes();
+
+ public static class KeyValidator {
+
+ public static String validateKey(final String key) {
+ // We used to validate the key by disallowing a handful of keywords, but this requirement no longer exists.
+ // Therefore this method simply verifies that the key is not empty.
+ if (key == null) {
+ throw new IllegalArgumentException("Invalid attribute key: null");
+ }
+ if (key.trim().isEmpty()) {
+ throw new IllegalArgumentException("Invalid attribute key: <Empty String>");
+ }
+ return key;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/flowfile/FlowFilePrioritizer.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/flowfile/FlowFilePrioritizer.java b/nifi-api/src/main/java/org/apache/nifi/flowfile/FlowFilePrioritizer.java
new file mode 100644
index 0000000..684f454
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/flowfile/FlowFilePrioritizer.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.flowfile;
+
+import java.util.Comparator;
+
+/**
+ * Provides a mechanism to prioritize flow file objects based on their
+ * attributes. The actual flow file content will not be available for comparison
+ * so if features of that content are necessary for prioritization it should be
+ * extracted to be used as an attribute of the flow file.
+ *
+ */
+public interface FlowFilePrioritizer extends Comparator<FlowFile> {
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/logging/ComponentLog.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/logging/ComponentLog.java b/nifi-api/src/main/java/org/apache/nifi/logging/ComponentLog.java
new file mode 100644
index 0000000..b4b3c6a
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/logging/ComponentLog.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.logging;
+
+/**
+ * <p>
+ * The ComponentLog provides a mechanism to ensure that all NiFi components are
+ * logging and reporting information in a consistent way. When messages are
+ * logged to the ComponentLog, each message has the following characteristics:
+ * </p>
+ *
+ * <ul>
+ * <li>
+ * The <code>toString()</code> of the component is automatically prepended to
+ * the message so that it is clear which component is providing the information.
+ * This is important, since a single component may have many different instances
+ * within the same NiFi instance.
+ * </li>
+ * <li>
+ * If the last value in an Object[] argument that is passed to the logger is a
+ * Throwable, then the logged message will include a <code>toString()</code> of
+ * the Throwable; in addition, if the component's logger is set to DEBUG level
+ * via the logback configuration, the Stacktrace will also be logged. This
+ * provides a mechanism to easily enable stacktraces in the logs when they are
+ * desired without filling the logs with unneeded stack traces for messages that
+ * end up occurring often.
+ * </li>
+ * <li>
+ * Any message that is logged with a Severity level that meets or exceeds the
+ * configured Bulletin Level for that component will also cause a Bulletin to be
+ * generated, so that the message is visible in the UI, allowing Dataflow
+ * Managers to understand that a problem exists and what the issue is.
+ * </li>
+ * </ul>
+ *
+ */
+public interface ComponentLog {
+
+ void warn(String msg, Throwable t);
+
+ void warn(String msg, Object[] os);
+
+ void warn(String msg, Object[] os, Throwable t);
+
+ void warn(String msg);
+
+ void trace(String msg, Throwable t);
+
+ void trace(String msg, Object[] os);
+
+ void trace(String msg);
+
+ void trace(String msg, Object[] os, Throwable t);
+
+ boolean isWarnEnabled();
+
+ boolean isTraceEnabled();
+
+ boolean isInfoEnabled();
+
+ boolean isErrorEnabled();
+
+ boolean isDebugEnabled();
+
+ void info(String msg, Throwable t);
+
+ void info(String msg, Object[] os);
+
+ void info(String msg);
+
+ void info(String msg, Object[] os, Throwable t);
+
+ String getName();
+
+ void error(String msg, Throwable t);
+
+ void error(String msg, Object[] os);
+
+ void error(String msg);
+
+ void error(String msg, Object[] os, Throwable t);
+
+ void debug(String msg, Throwable t);
+
+ void debug(String msg, Object[] os);
+
+ void debug(String msg, Object[] os, Throwable t);
+
+ void debug(String msg);
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/logging/LogLevel.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/logging/LogLevel.java b/nifi-api/src/main/java/org/apache/nifi/logging/LogLevel.java
new file mode 100644
index 0000000..0c9b64b
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/logging/LogLevel.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.logging;
+
+public enum LogLevel {
+
+ TRACE,
+ DEBUG,
+ INFO,
+ WARN,
+ ERROR,
+ FATAL;
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/logging/ProcessorLog.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/logging/ProcessorLog.java b/nifi-api/src/main/java/org/apache/nifi/logging/ProcessorLog.java
new file mode 100644
index 0000000..a90ee26
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/logging/ProcessorLog.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.logging;
+
+/**
+ * The ProcessorLog is an extension of ComponentLog but provides no additional
+ * functionality. It exists because ProcessorLog was created first, but when
+ * Controller Services and Reporting Tasks began to be used more heavily loggers
+ * were needed for them as well. We did not want to return a ProcessorLog to a
+ * ControllerService or a ReportingTask, so all of the methods were moved to a
+ * higher interface named ComponentLog. However, we kept the ProcessorLog
+ * interface around in order to maintain backward compatibility.
+ */
+public interface ProcessorLog extends ComponentLog {
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/processor/AbstractProcessor.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/AbstractProcessor.java b/nifi-api/src/main/java/org/apache/nifi/processor/AbstractProcessor.java
new file mode 100644
index 0000000..7ff568e
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/processor/AbstractProcessor.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor;
+
+import org.apache.nifi.processor.exception.ProcessException;
+
+public abstract class AbstractProcessor extends AbstractSessionFactoryProcessor {
+
+ @Override
+ public final void onTrigger(final ProcessContext context, final ProcessSessionFactory sessionFactory) throws ProcessException {
+ final ProcessSession session = sessionFactory.createSession();
+ try {
+ onTrigger(context, session);
+ session.commit();
+ } catch (final Throwable t) {
+ getLogger().error("{} failed to process due to {}; rolling back session", new Object[]{this, t});
+ session.rollback(true);
+ throw t;
+ }
+ }
+
+ public abstract void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException;
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/processor/AbstractSessionFactoryProcessor.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/AbstractSessionFactoryProcessor.java b/nifi-api/src/main/java/org/apache/nifi/processor/AbstractSessionFactoryProcessor.java
new file mode 100644
index 0000000..2695dcd
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/processor/AbstractSessionFactoryProcessor.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor;
+
+import java.util.Collections;
+import java.util.Set;
+
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.annotation.lifecycle.OnUnscheduled;
+import org.apache.nifi.components.AbstractConfigurableComponent;
+import org.apache.nifi.controller.ControllerServiceLookup;
+import org.apache.nifi.logging.ProcessorLog;
+
+/**
+ * <p>
+ * Provides a standard partial implementation of a {@link Processor}. This
+ * implementation provides default behavior and various convenience hooks for
+ * processing.</p>
+ *
+ * <p>
+ * Implementation/Design note: This class follows the open/closed principle in a
+ * fairly strict manner meaning that subclasses are free to customize behavior
+ * in specifically designed points exclusively. If greater flexibility is
+ * necessary then it is still possible to simply implement the {@link Processor}
+ * interface.</p>
+ *
+ * <p>
+ * Thread safe</p>
+ *
+ */
+public abstract class AbstractSessionFactoryProcessor extends AbstractConfigurableComponent implements Processor {
+
+ private String identifier;
+ private ProcessorLog logger;
+ private volatile boolean scheduled = false;
+ private ControllerServiceLookup serviceLookup;
+ private String description;
+
+ @Override
+ public final void initialize(final ProcessorInitializationContext context) {
+ identifier = context.getIdentifier();
+ logger = context.getLogger();
+ serviceLookup = context.getControllerServiceLookup();
+ init(context);
+
+ description = getClass().getSimpleName() + "[id=" + identifier + "]";
+ }
+
+ /**
+ * @return the {@link ControllerServiceLookup} that was passed to the
+ * {@link #init(ProcessorInitializationContext)} method
+ */
+ protected final ControllerServiceLookup getControllerServiceLookup() {
+ return serviceLookup;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return Collections.emptySet();
+ }
+
+ protected final ProcessorLog getLogger() {
+ return logger;
+ }
+
+ /**
+ * Provides subclasses the ability to perform initialization logic
+ *
+ * @param context in which to perform initialization
+ */
+ protected void init(final ProcessorInitializationContext context) {
+ // Provided for subclasses to override
+ }
+
+ /**
+ * @return <code>true</code> if the processor is scheduled to run,
+ * <code>false</code> otherwise
+ */
+ protected final boolean isScheduled() {
+ return scheduled;
+ }
+
+ @OnScheduled
+ public final void updateScheduledTrue() {
+ scheduled = true;
+ }
+
+ @OnUnscheduled
+ public final void updateScheduledFalse() {
+ scheduled = false;
+ }
+
+ @Override
+ public final String getIdentifier() {
+ return identifier;
+ }
+
+ @Override
+ public String toString() {
+ return description;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/processor/DataUnit.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/DataUnit.java b/nifi-api/src/main/java/org/apache/nifi/processor/DataUnit.java
new file mode 100644
index 0000000..4980b97
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/processor/DataUnit.java
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public enum DataUnit {
+
+ /**
+ * Bytes
+ */
+ B {
+ @Override
+ public double toB(double value) {
+ return value;
+ }
+
+ @Override
+ public double toKB(double value) {
+ return value / POWERS[1];
+ }
+
+ @Override
+ public double toMB(double value) {
+ return value / POWERS[2];
+ }
+
+ @Override
+ public double toGB(double value) {
+ return value / POWERS[3];
+ }
+
+ @Override
+ public double toTB(double value) {
+ return value / POWERS[4];
+ }
+
+ @Override
+ public double convert(double sourceSize, DataUnit sourceUnit) {
+ return sourceUnit.toB(sourceSize);
+ }
+ },
+ /**
+ * Kilobytes
+ */
+ KB {
+ @Override
+ public double toB(double value) {
+ return value * POWERS[1];
+ }
+
+ @Override
+ public double toKB(double value) {
+ return value;
+ }
+
+ @Override
+ public double toMB(double value) {
+ return value / POWERS[1];
+ }
+
+ @Override
+ public double toGB(double value) {
+ return value / POWERS[2];
+ }
+
+ @Override
+ public double toTB(double value) {
+ return value / POWERS[3];
+ }
+
+ @Override
+ public double convert(double sourceSize, DataUnit sourceUnit) {
+ return sourceUnit.toKB(sourceSize);
+ }
+ },
+ /**
+ * Megabytes
+ */
+ MB {
+ @Override
+ public double toB(double value) {
+ return value * POWERS[2];
+ }
+
+ @Override
+ public double toKB(double value) {
+ return value * POWERS[1];
+ }
+
+ @Override
+ public double toMB(double value) {
+ return value;
+ }
+
+ @Override
+ public double toGB(double value) {
+ return value / POWERS[1];
+ }
+
+ @Override
+ public double toTB(double value) {
+ return value / POWERS[2];
+ }
+
+ @Override
+ public double convert(double sourceSize, DataUnit sourceUnit) {
+ return sourceUnit.toMB(sourceSize);
+ }
+ },
+ /**
+ * Gigabytes
+ */
+ GB {
+ @Override
+ public double toB(double value) {
+ return value * POWERS[3];
+ }
+
+ @Override
+ public double toKB(double value) {
+ return value * POWERS[2];
+ }
+
+ @Override
+ public double toMB(double value) {
+ return value * POWERS[1];
+ }
+
+ @Override
+ public double toGB(double value) {
+ return value;
+ }
+
+ @Override
+ public double toTB(double value) {
+ return value / POWERS[1];
+ }
+
+ @Override
+ public double convert(double sourceSize, DataUnit sourceUnit) {
+ return sourceUnit.toGB(sourceSize);
+ }
+ },
+ /**
+ * Terabytes
+ */
+ TB {
+ @Override
+ public double toB(double value) {
+ return value * POWERS[4];
+ }
+
+ @Override
+ public double toKB(double value) {
+ return value * POWERS[3];
+ }
+
+ @Override
+ public double toMB(double value) {
+ return value * POWERS[2];
+ }
+
+ @Override
+ public double toGB(double value) {
+ return value * POWERS[1];
+ }
+
+ @Override
+ public double toTB(double value) {
+ return value;
+ }
+
+ @Override
+ public double convert(double sourceSize, DataUnit sourceUnit) {
+ return sourceUnit.toTB(sourceSize);
+ }
+ };
+
+ public double convert(final double sourceSize, final DataUnit sourceUnit) {
+ throw new AbstractMethodError();
+ }
+
+ public double toB(double size) {
+ throw new AbstractMethodError();
+ }
+
+ public double toKB(double size) {
+ throw new AbstractMethodError();
+ }
+
+ public double toMB(double size) {
+ throw new AbstractMethodError();
+ }
+
+ public double toGB(double size) {
+ throw new AbstractMethodError();
+ }
+
+ public double toTB(double size) {
+ throw new AbstractMethodError();
+ }
+
+ public static final double[] POWERS = {1,
+ 1024D,
+ 1024 * 1024D,
+ 1024 * 1024 * 1024D,
+ 1024 * 1024 * 1024 * 1024D};
+
+ public static final String DATA_SIZE_REGEX = "(\\d+(?:\\.\\d+)?)\\s*(B|KB|MB|GB|TB)";
+ public static final Pattern DATA_SIZE_PATTERN = Pattern.compile(DATA_SIZE_REGEX);
+
+ public static Double parseDataSize(final String value, final DataUnit units) {
+ if (value == null) {
+ return null;
+ }
+
+ final Matcher matcher = DATA_SIZE_PATTERN.matcher(value.toUpperCase());
+ if (!matcher.find()) {
+ throw new IllegalArgumentException("Invalid data size: " + value);
+ }
+
+ final String sizeValue = matcher.group(1);
+ final String unitValue = matcher.group(2);
+
+ final DataUnit sourceUnit = DataUnit.valueOf(unitValue);
+ final double size = Double.parseDouble(sizeValue);
+ return units.convert(size, sourceUnit);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/processor/FlowFileFilter.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/FlowFileFilter.java b/nifi-api/src/main/java/org/apache/nifi/processor/FlowFileFilter.java
new file mode 100644
index 0000000..3bd6546
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/processor/FlowFileFilter.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor;
+
+import org.apache.nifi.flowfile.FlowFile;
+
+/**
+ * <p>
+ * FlowFileFilter provides a mechanism for selectively choosing which FlowFiles
+ * to obtain from a Processor's incoming connections.
+ * </p>
+ *
+ * <p>
+ * Implementations of this interface need not be thread-safe.
+ * </p>
+ */
+public interface FlowFileFilter {
+
+ /**
+ * Indicates whether or not the given FlowFile should be selected and
+ * whether or not the Processor is interested in filtering additional
+ * FlowFiles
+ *
+ * @param flowFile to apply the filter to
+ * @return true if the given FlowFile should be selected and
+ * if Processor is interested in filtering additional
+ * FlowFiles
+ */
+ FlowFileFilterResult filter(FlowFile flowFile);
+
+ /**
+ * Provides a result type to indicate whether or not a FlowFile should be
+ * selected
+ */
+ public static enum FlowFileFilterResult {
+
+ /**
+ * Indicates that a FlowFile should be returned to the Processor to be
+ * processed and that additional FlowFiles should be processed by this
+ * filter.
+ */
+ ACCEPT_AND_CONTINUE(true, true),
+ /**
+ * Indicates that a FlowFile should be returned to the Processor to be
+ * processed and that this is the last FlowFile that should be processed
+ * by this filter.
+ */
+ ACCEPT_AND_TERMINATE(false, true),
+ /**
+ * Indicates that a FlowFile should not be processed by the Processor at
+ * this time but that additional FlowFiles should be processed by this
+ * filter.
+ */
+ REJECT_AND_CONTINUE(true, false),
+ /**
+ * Indicates that a FlowFile should not be processed by the Processor at
+ * this time and that no additional FlowFiles should be processed
+ * either.
+ */
+ REJECT_AND_TERMINATE(false, false);
+
+ private final boolean continueProcessing;
+ private final boolean accept;
+
+ private FlowFileFilterResult(final boolean continueProcessing, final boolean accept) {
+ this.continueProcessing = continueProcessing;
+ this.accept = accept;
+ }
+
+ public boolean isAccept() {
+ return accept;
+ }
+
+ public boolean isContinue() {
+ return continueProcessing;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/processor/ProcessContext.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/ProcessContext.java b/nifi-api/src/main/java/org/apache/nifi/processor/ProcessContext.java
new file mode 100644
index 0000000..c61a318
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/processor/ProcessContext.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor;
+
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.PropertyValue;
+import org.apache.nifi.controller.ControllerServiceLookup;
+
+/**
+ * <p>
+ * Provides a bridge between a Processor and the NiFi Framework
+ * </p>
+ *
+ * <p>
+ * <b>Note: </b>Implementations of this interface are NOT necessarily
+ * thread-safe.
+ * </p>
+ */
+public interface ProcessContext {
+
+ /**
+ * Retrieves the current value set for the given descriptor, if a value is
+ * set - else uses the descriptor to determine the appropriate default value
+ *
+ * @param descriptor to lookup the value of
+ * @return the property value of the given descriptor
+ */
+ PropertyValue getProperty(PropertyDescriptor descriptor);
+
+ /**
+ * Retrieves the current value set for the given descriptor, if a value is
+ * set - else uses the descriptor to determine the appropriate default value
+ *
+ * @param propertyName of the property to lookup the value for
+ * @return property value as retrieved by property name
+ */
+ PropertyValue getProperty(String propertyName);
+
+ /**
+ * Creates and returns a {@link PropertyValue} object that can be used for
+ * evaluating the value of the given String
+ *
+ * @param rawValue the raw input before any property evaluation has occurred
+ * @return a {@link PropertyValue} object that can be used for
+ * evaluating the value of the given String
+ */
+ PropertyValue newPropertyValue(String rawValue);
+
+ /**
+ * <p>
+ * Causes the Processor not to be scheduled for some pre-configured amount
+ * of time. The duration of time for which the processor will not be
+ * scheduled is configured in the same manner as the processor's scheduling
+ * period.
+ * </p>
+ *
+ * <p>
+ * <b>Note: </b>This is NOT a blocking call and does not suspend execution
+ * of the current thread.
+ * </p>
+ */
+ void yield();
+
+ /**
+ * @return the maximum number of threads that may be executing this
+ * processor's code at any given time
+ */
+ int getMaxConcurrentTasks();
+
+ /**
+ * @return the annotation data configured for this processor
+ */
+ String getAnnotationData();
+
+ /**
+ * @return a Map of all PropertyDescriptors to their configured values. This
+ * Map may or may not be modifiable, but modifying its values will not
+ * change the values of the processor's properties
+ */
+ Map<PropertyDescriptor, String> getProperties();
+
+ /**
+ * Encrypts the given value using the password provided in the NiFi
+ * Properties
+ *
+ * @param unencrypted plaintext value
+ * @return encrypted value
+ */
+ String encrypt(String unencrypted);
+
+ /**
+ * Decrypts the given value using the password provided in the NiFi
+ * Properties
+ *
+ * @param encrypted the encrypted value
+ * @return the plaintext value
+ */
+ String decrypt(String encrypted);
+
+ /**
+ * @return a {@code ControllerServiceLookup} that can be used to obtain a
+ * Controller Service
+ */
+ ControllerServiceLookup getControllerServiceLookup();
+
+ /**
+ * @return the set of all relationships for which space is available to
+ * receive new objects
+ */
+ Set<Relationship> getAvailableRelationships();
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/processor/ProcessSession.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/ProcessSession.java b/nifi-api/src/main/java/org/apache/nifi/processor/ProcessSession.java
new file mode 100644
index 0000000..ed46d68
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/processor/ProcessSession.java
@@ -0,0 +1,736 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Path;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.exception.FlowFileAccessException;
+import org.apache.nifi.processor.exception.FlowFileHandlingException;
+import org.apache.nifi.processor.exception.MissingFlowFileException;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.io.InputStreamCallback;
+import org.apache.nifi.processor.io.OutputStreamCallback;
+import org.apache.nifi.processor.io.StreamCallback;
+import org.apache.nifi.provenance.ProvenanceReporter;
+
+/**
+ * <p>
+ * A process session encompasses all the behaviors a processor can perform to
+ * obtain, clone, read, modify remove FlowFiles in an atomic unit. A process
+ * session is always tied to a single processor at any one time and ensures no
+ * FlowFile can ever be accessed by any more than one processor at a given time.
+ * The session also ensures that all FlowFiles are always accounted for. The
+ * creator of a ProcessSession is always required to manage the session.</p>
+ *
+ * <p>
+ * A session is not considered thread safe. The session supports a unit of work
+ * that is either committed or rolled back</p>
+ *
+ * <p>
+ * As noted on specific methods and for specific exceptions automated rollback
+ * will occur to ensure consistency of the repository. However, several
+ * situations can result in exceptions yet not cause automated rollback. In
+ * these cases the consistency of the repository will be retained but callers
+ * will be able to indicate whether it should result in rollback or continue on
+ * toward a commit.</p>
+ *
+ * <p>
+ * A process session instance may be used continuously. That is, after each
+ * commit or rollback, the session can be used again.</p>
+ *
+ */
+public interface ProcessSession {
+
+ /**
+ * <p>
+ * Commits the current session ensuring all operations against FlowFiles
+ * within this session are atomically persisted. All FlowFiles operated on
+ * within this session must be accounted for by transfer or removal or the
+ * commit will fail.</p>
+ *
+ * <p>
+ * As soon as the commit completes the session is again ready to be used</p>
+ *
+ * @throws IllegalStateException if detected that this method is being
+ * called from within a callback of another method in this session.
+ * @throws FlowFileHandlingException if not all FlowFiles acted upon within
+ * this session are accounted for by user code such that they have a
+ * transfer identified or where marked for removal. Automated rollback
+ * occurs.
+ * @throws ProcessException if some general fault occurs while persisting
+ * the session. Initiates automatic rollback. The root cause can be obtained
+ * via <code>Exception.getCause()</code>
+ */
+ void commit();
+
+ /**
+ * Reverts any changes made during this session. All FlowFiles are restored
+ * back to their initial session state and back to their original queues. If
+ * this session is already committed or rolled back then no changes will
+ * occur. This method can be called any number of times. Calling this method
+ * is identical to calling {@link #rollback(boolean)} passing
+ * <code>false</code> as the parameter.
+ */
+ void rollback();
+
+ /**
+ * Reverts any changes made during this session. All FlowFiles are restored
+ * back to their initial session state and back to their original queues,
+ * after optionally being penalized. If this session is already committed or
+ * rolled back then no changes will occur. This method can be called any
+ * number of times.
+ *
+ * @param penalize whether or not the FlowFiles that are being restored back
+ * to their queues should be penalized
+ */
+ void rollback(boolean penalize);
+
+ /**
+ * Adjusts counter data for the given counter name and takes care of
+ * registering the counter if not already present. The adjustment occurs
+ * only if and when the ProcessSession is committed.
+ *
+ * @param name the name of the counter
+ * @param delta the delta by which to modify the counter (+ or -)
+ * @param immediate if true, the counter will be updated immediately,
+ * without regard to whether the ProcessSession is commit or rolled back;
+ * otherwise, the counter will be incremented only if and when the
+ * ProcessSession is committed.
+ */
+ void adjustCounter(String name, long delta, boolean immediate);
+
+ /**
+ * @return FlowFile that is next highest priority FlowFile to process.
+ * Otherwise returns null.
+ */
+ FlowFile get();
+
+ /**
+ * Returns up to <code>maxResults</code> FlowFiles from the work queue. If
+ * no FlowFiles are available, returns an empty list. Will not return null.
+ * If multiple incoming queues are present, the behavior is unspecified in
+ * terms of whether all queues or only a single queue will be polled in a
+ * single call.
+ *
+ * @param maxResults the maximum number of FlowFiles to return
+ * @return up to <code>maxResults</code> FlowFiles from the work queue. If
+ * no FlowFiles are available, returns an empty list. Will not return null.
+ * @throws IllegalArgumentException if <code>maxResults</code> is less than
+ * 0
+ */
+ List<FlowFile> get(int maxResults);
+
+ /**
+ * <p>
+ * Returns all FlowFiles from all of the incoming queues for which the given
+ * {@link FlowFileFilter} indicates should be accepted. Calls to this method
+ * provide exclusive access to the underlying queues. I.e., no other thread
+ * will be permitted to pull FlowFiles from this Processor's queues or add
+ * FlowFiles to this Processor's incoming queues until this method call has
+ * returned.
+ * </p>
+ *
+ * @param filter to limit which flow files are returned
+ * @return all FlowFiles from all of the incoming queues for which the given
+ * {@link FlowFileFilter} indicates should be accepted.
+ */
+ List<FlowFile> get(FlowFileFilter filter);
+
+ /**
+ * @return the QueueSize that represents the number of FlowFiles and their
+ * combined data size for all FlowFiles waiting to be processed by the
+ * Processor that owns this ProcessSession, regardless of which Connection
+ * the FlowFiles live on
+ */
+ QueueSize getQueueSize();
+
+ /**
+ * Creates a new FlowFile in the repository with no content and without any
+ * linkage to a parent FlowFile. This method is appropriate only when data
+ * is received or created from an external system. Otherwise, this method
+ * should be avoided and should instead use {@link #create(FlowFile)} or
+ * {@see #create(Collection)}.
+ *
+ * When this method is used, a Provenance CREATE or RECEIVE Event should be
+ * generated. See the {@link #getProvenanceReporter()} method and
+ * {@link ProvenanceReporter} class for more information
+ *
+ * @return newly created FlowFile
+ */
+ FlowFile create();
+
+ /**
+ * Creates a new FlowFile in the repository with no content but with a
+ * parent linkage to <code>parent</code>. The newly created FlowFile will
+ * inherit all of the parent's attributes except for the UUID. This method
+ * will automatically generate a Provenance FORK event or a Provenance JOIN
+ * event, depending on whether or not other FlowFiles are generated from the
+ * same parent before the ProcessSession is committed.
+ *
+ * @param parent to base the new flowfile on
+ * @return newly created flowfile
+ */
+ FlowFile create(FlowFile parent);
+
+ /**
+ * Creates a new FlowFile in the repository with no content but with a
+ * parent linkage to the FlowFiles specified by the parents Collection. The
+ * newly created FlowFile will inherit all of the attributes that are in
+ * common to all parents (except for the UUID, which will be in common if
+ * only a single parent exists). This method will automatically generate a
+ * Provenance JOIN event.
+ *
+ * @param parents which the new flowfile should inherit shared attributes from
+ * @return new flowfile
+ */
+ FlowFile create(Collection<FlowFile> parents);
+
+ /**
+ * Creates a new FlowFile that is a clone of the given FlowFile as of the
+ * time this is called, both in content and attributes. This method
+ * automatically emits a Provenance CLONE Event.
+ *
+ * @param example FlowFile to be the source of cloning - given FlowFile must
+ * be a part of the given session
+ * @return FlowFile that is a clone of the given example
+ * @throws IllegalStateException if detected that this method is being
+ * called from within a callback of another method in this session and for
+ * the given FlowFile(s)
+ * @throws FlowFileHandlingException if the given FlowFile is already
+ * transferred or removed or doesn't belong to this session. Automatic
+ * rollback will occur.
+ * @throws MissingFlowFileException if the given FlowFile content cannot be
+ * found. The FlowFile should no longer be reference, will be internally
+ * destroyed, and the session is automatically rolled back and what is left
+ * of the FlowFile is destroyed.
+ * @throws FlowFileAccessException if some IO problem occurs accessing
+ * FlowFile content
+ * @throws NullPointerException if the argument null
+ */
+ FlowFile clone(FlowFile example);
+
+ /**
+ * Creates a new FlowFile whose parent is the given FlowFile. The content of
+ * the new FlowFile will be a subset of the byte sequence of the given
+ * FlowFile starting at the specified offset and with the length specified.
+ * The new FlowFile will contain all of the attributes of the original. This
+ * method automatically emits a Provenance FORK Event (or a Provenance CLONE
+ * Event, if the offset is 0 and the size is exactly equal to the size of
+ * the example FlowFile).
+ *
+ * @param parent to base the new flowfile attributes on
+ * @param offset of the parent flowfile to base the child flowfile content on
+ * @param size of the new flowfile from the offset
+ * @return a FlowFile with the specified size whose parent is first argument
+ * to this function
+ *
+ * @throws IllegalStateException if detected that this method is being
+ * called from within a callback of another method in this session and for
+ * the given FlowFile
+ * @throws FlowFileHandlingException if the given FlowFile is already
+ * transferred or removed or doesn't belong to this session, or if the
+ * specified offset + size exceeds that of the size of the parent FlowFile.
+ * Automatic rollback will occur.
+ * @throws MissingFlowFileException if the given FlowFile content cannot be
+ * found. The FlowFile should no longer be reference, will be internally
+ * destroyed, and the session is automatically rolled back and what is left
+ * of the FlowFile is destroyed.
+ */
+ FlowFile clone(FlowFile parent, long offset, long size);
+
+ /**
+ * Sets a penalty for the given FlowFile which will make it unavailable to
+ * be operated on any further during the penalty period.
+ *
+ * @param flowFile to penalize
+ * @return FlowFile the new FlowFile reference to use
+ * @throws IllegalStateException if detected that this method is being
+ * called from within a callback of another method in this session and for
+ * the given FlowFile(s)
+ * @throws FlowFileHandlingException if the given FlowFile is already
+ * transferred or removed or doesn't belong to this session. Automatic
+ * rollback will occur.
+ * @throws NullPointerException if the argument null
+ */
+ FlowFile penalize(FlowFile flowFile);
+
+ /**
+ * Updates the given FlowFiles attributes with the given key/value pair. If
+ * the key is named {@code uuid}, this attribute will be ignored.
+ *
+ * @param flowFile to update
+ * @param key of attribute
+ * @param value of attribute
+ * @return FlowFile the updated FlowFile
+ * @throws FlowFileHandlingException if the given FlowFile is already
+ * transferred or removed or doesn't belong to this session. Automatic
+ * rollback will occur.
+ * @throws NullPointerException if an argument is null
+ */
+ FlowFile putAttribute(FlowFile flowFile, String key, String value);
+
+ /**
+ * Updates the given FlowFiles attributes with the given key/value pairs. If
+ * the map contains a key named {@code uuid}, this attribute will be
+ * ignored.
+ *
+ * @param flowFile to update
+ * @param attributes the attributes to add to the given FlowFile
+ * @return FlowFile the updated FlowFile
+ * @throws IllegalStateException if detected that this method is being
+ * called from within a callback of another method in this session and for
+ * the given FlowFile(s)
+ * @throws FlowFileHandlingException if the given FlowFile is already
+ * transferred or removed or doesn't belong to this session. Automatic
+ * rollback will occur.
+ * @throws NullPointerException if an argument is null
+ */
+ FlowFile putAllAttributes(FlowFile flowFile, Map<String, String> attributes);
+
+ /**
+ * Removes the given FlowFile attribute with the given key. If the key is
+ * named {@code uuid}, this method will return the same FlowFile without
+ * removing any attribute.
+ *
+ * @param flowFile to update
+ * @param key of attribute
+ * @return FlowFile the updated FlowFile
+ * @throws IllegalStateException if detected that this method is being
+ * called from within a callback of another method in this session and for
+ * the given FlowFile(s)
+ * @throws FlowFileHandlingException if the given FlowFile is already
+ * transferred or removed or doesn't belong to this session. Automatic
+ * rollback will occur.
+ * @throws NullPointerException if the argument null
+ */
+ FlowFile removeAttribute(FlowFile flowFile, String key);
+
+ /**
+ * Removes the attributes with the given keys from the given FlowFile. If
+ * the set of keys contains the value {@code uuid}, this key will be ignored
+ *
+ * @param flowFile to update
+ * @param keys of attribute
+ * @return FlowFile the updated FlowFile
+ * @throws IllegalStateException if detected that this method is being
+ * called from within a callback of another method in this session and for
+ * the given FlowFile(s)
+ * @throws FlowFileHandlingException if the given FlowFile is already
+ * transferred or removed or doesn't belong to this session. Automatic
+ * rollback will occur.
+ * @throws NullPointerException if the argument null
+ */
+ FlowFile removeAllAttributes(FlowFile flowFile, Set<String> keys);
+
+ /**
+ * Remove all attributes from the given FlowFile that have keys which match
+ * the given pattern. If the pattern matches the key {@code uuid}, this key
+ * will not be removed.
+ *
+ * @param flowFile to update
+ * @param keyPattern may be null; if supplied is matched against each of the
+ * FlowFile attribute keys
+ * @return FlowFile containing only attributes which did not meet the key
+ * pattern
+ */
+ FlowFile removeAllAttributes(FlowFile flowFile, Pattern keyPattern);
+
+ /**
+ * Transfers the given FlowFile to the appropriate destination processor
+ * work queue(s) based on the given relationship. If the relationship leads
+ * to more than one destination the state of the FlowFile is replicated such
+ * that each destination receives an exact copy of the FlowFile though each
+ * will have its own unique identity. The destination processors will not be
+ * able to operate on the given FlowFile until this session is committed or
+ * until the ownership of the session is migrated to another processor. If
+ * ownership of the session is passed to a destination processor then that
+ * destination processor will have immediate visibility of the transferred
+ * FlowFiles within the session.
+ *
+ * @param flowFile to transfer
+ * @param relationship to transfer to
+ * @throws IllegalStateException if detected that this method is being
+ * called from within a callback of another method in this session and for
+ * the given FlowFile(s)
+ * @throws FlowFileHandlingException if the given FlowFile is already
+ * transferred or removed or doesn't belong to this session. Automatic
+ * rollback will occur.
+ * @throws NullPointerException if the argument null
+ * @throws IllegalArgumentException if given relationship is not a known or
+ * registered relationship
+ */
+ void transfer(FlowFile flowFile, Relationship relationship);
+
+ /**
+ * Transfers the given FlowFile back to the work queue from which it was
+ * pulled. The processor will not be able to operate on the given FlowFile
+ * until this session is committed. Any modifications that have been made to
+ * the FlowFile will be maintained. FlowFiles that are created by the
+ * processor cannot be transferred back to themselves via this method.
+ *
+ * @param flowFile to transfer
+ * @throws IllegalStateException if detected that this method is being
+ * called from within a callback of another method in this session and for
+ * the given FlowFile(s)
+ * @throws FlowFileHandlingException if the given FlowFile is already
+ * transferred or removed or doesn't belong to this session. Automatic
+ * rollback will occur.
+ * @throws IllegalArgumentException if the FlowFile was created by this
+ * processor
+ * @throws NullPointerException if the argument null
+ */
+ void transfer(FlowFile flowFile);
+
+ /**
+ * Transfers the given FlowFiles back to the work queues from which the
+ * FlowFiles were pulled. The processor will not be able to operate on the
+ * given FlowFile until this session is committed. Any modifications that
+ * have been made to the FlowFile will be maintained. FlowFiles that are
+ * created by the processor cannot be transferred back to themselves via
+ * this method.
+ *
+ * @param flowFiles to transfer
+ * @throws IllegalStateException if detected that this method is being
+ * called from within a callback of another method in this session and for
+ * the given FlowFile(s)
+ * @throws FlowFileHandlingException if the given FlowFiles are already
+ * transferred or removed or don't belong to this session. Automatic
+ * rollback will occur.
+ * @throws IllegalArgumentException if the FlowFile was created by this
+ * processor
+ * @throws NullPointerException if the argument null
+ */
+ void transfer(Collection<FlowFile> flowFiles);
+
+ /**
+ * Transfers the given FlowFile to the appropriate destination processor
+ * work queue(s) based on the given relationship. If the relationship leads
+ * to more than one destination the state of the FlowFile is replicated such
+ * that each destination receives an exact copy of the FlowFile though each
+ * will have its own unique identity. The destination processors will not be
+ * able to operate on the given FlowFile until this session is committed or
+ * until the ownership of the session is migrated to another processor. If
+ * ownership of the session is passed to a destination processor then that
+ * destination processor will have immediate visibility of the transferred
+ * FlowFiles within the session.
+ *
+ * @param flowFiles to transfer
+ * @param relationship to transfer to
+ * @throws IllegalStateException if detected that this method is being
+ * called from within a callback of another method in this session and for
+ * the given FlowFile(s)
+ * @throws FlowFileHandlingException if the given FlowFile is already
+ * transferred or removed or doesn't belong to this session. Automatic
+ * rollback will occur.
+ * @throws NullPointerException if the argument null
+ * @throws IllegalArgumentException if given relationship is not a known or
+ * registered relationship
+ */
+ void transfer(Collection<FlowFile> flowFiles, Relationship relationship);
+
+ /**
+ * Ends the managed persistence for the given FlowFile. The persistent
+ * attributes for the FlowFile are deleted and so is the content assuming
+ * nothing else references it and this FlowFile will no longer be available
+ * for further operation.
+ *
+ * @param flowFile to remove
+ * @throws IllegalStateException if detected that this method is being
+ * called from within a callback of another method in this session and for
+ * the given FlowFile(s)
+ * @throws FlowFileHandlingException if the given FlowFile is already
+ * transferred or removed or doesn't belong to this session. Automatic
+ * rollback will occur.
+ */
+ void remove(FlowFile flowFile);
+
+ /**
+ * Ends the managed persistence for the given FlowFiles. The persistent
+ * attributes for the FlowFile are deleted and so is the content assuming
+ * nothing else references it and this FlowFile will no longer be available
+ * for further operation.
+ *
+ * @param flowFiles to remove
+ * @throws IllegalStateException if detected that this method is being
+ * called from within a callback of another method in this session and for
+ * the given FlowFile(s)
+ * @throws FlowFileHandlingException if any of the given FlowFile is already
+ * transferred or removed or doesn't belong to this session. Automatic
+ * rollback will occur.
+ */
+ void remove(Collection<FlowFile> flowFiles);
+
+ /**
+ * Executes the given callback against the contents corresponding to the
+ * given FlowFile.
+ *
+ * <i>Note</i>: The OutputStream provided to the given OutputStreamCallback
+ * will not be accessible once this method has completed its execution.
+ *
+ * @param source flowfile to retrieve content of
+ * @param reader that will be called to read the flowfile content
+ * @throws IllegalStateException if detected that this method is being
+ * called from within a callback of another method in this session and for
+ * the given FlowFile(s)
+ * @throws FlowFileHandlingException if the given FlowFile is already
+ * transferred or removed or doesn't belong to this session. Automatic
+ * rollback will occur.
+ * @throws MissingFlowFileException if the given FlowFile content cannot be
+ * found. The FlowFile should no longer be reference, will be internally
+ * destroyed, and the session is automatically rolled back and what is left
+ * of the FlowFile is destroyed.
+ * @throws FlowFileAccessException if some IO problem occurs accessing
+ * FlowFile content; if an attempt is made to access the InputStream
+ * provided to the given InputStreamCallback after this method completed its
+ * execution
+ */
+ void read(FlowFile source, InputStreamCallback reader) throws FlowFileAccessException;
+
+ /**
+ * Combines the content of all given source FlowFiles into a single given
+ * destination FlowFile.
+ *
+ * @param sources the flowfiles to merge
+ * @param destination the flowfile to use as the merged result
+ * @return updated destination FlowFile (new size, etc...)
+ * @throws IllegalStateException if detected that this method is being
+ * called from within a callback of another method in this session and for
+ * the given FlowFile(s)
+ * @throws IllegalArgumentException if the given destination is contained
+ * within the sources
+ * @throws FlowFileHandlingException if the given FlowFile is already
+ * transferred or removed or doesn't belong to this session. Automatic
+ * rollback will occur.
+ * @throws MissingFlowFileException if the given FlowFile content cannot be
+ * found. The FlowFile should no longer be reference, will be internally
+ * destroyed, and the session is automatically rolled back and what is left
+ * of the FlowFile is destroyed.
+ * @throws FlowFileAccessException if some IO problem occurs accessing
+ * FlowFile content. The state of the destination will be as it was prior to
+ * this call.
+ */
+ FlowFile merge(Collection<FlowFile> sources, FlowFile destination);
+
+ /**
+ * Combines the content of all given source FlowFiles into a single given
+ * destination FlowFile.
+ *
+ * @param sources to merge together
+ * @param destination to merge to
+ * @param header bytes that will be added to the beginning of the merged
+ * output. May be null or empty.
+ * @param footer bytes that will be added to the end of the merged output.
+ * May be null or empty.
+ * @param demarcator bytes that will be placed in between each object merged
+ * together. May be null or empty.
+ * @return updated destination FlowFile (new size, etc...)
+ * @throws IllegalStateException if detected that this method is being
+ * called from within a callback of another method in this session and for
+ * the given FlowFile(s)
+ * @throws IllegalArgumentException if the given destination is contained
+ * within the sources
+ * @throws FlowFileHandlingException if the given FlowFile is already
+ * transferred or removed or doesn't belong to this session. Automatic
+ * rollback will occur.
+ * @throws MissingFlowFileException if the given FlowFile content cannot be
+ * found. The FlowFile should no longer be reference, will be internally
+ * destroyed, and the session is automatically rolled back and what is left
+ * of the FlowFile is destroyed.
+ * @throws FlowFileAccessException if some IO problem occurs accessing
+ * FlowFile content. The state of the destination will be as it was prior to
+ * this call.
+ */
+ FlowFile merge(Collection<FlowFile> sources, FlowFile destination, byte[] header, byte[] footer, byte[] demarcator);
+
+ /**
+ * Executes the given callback against the content corresponding to the
+ * given FlowFile.
+ *
+ * <i>Note</i>: The OutputStream provided to the given OutputStreamCallback
+ * will not be accessible once this method has completed its execution.
+ *
+ * @param source to write to
+ * @param writer used to write new content
+ * @return updated FlowFile
+ * @throws IllegalStateException if detected that this method is being
+ * called from within a callback of another method in this session and for
+ * the given FlowFile(s)
+ * @throws FlowFileHandlingException if the given FlowFile is already
+ * transferred or removed or doesn't belong to this session. Automatic
+ * rollback will occur.
+ * @throws MissingFlowFileException if the given FlowFile content cannot be
+ * found. The FlowFile should no longer be referenced, will be internally
+ * destroyed, and the session is automatically rolled back and what is left
+ * of the FlowFile is destroyed.
+ * @throws FlowFileAccessException if some IO problem occurs accessing
+ * FlowFile content; if an attempt is made to access the OutputStream
+ * provided to the given OutputStreamCallaback after this method completed
+ * its execution
+ */
+ FlowFile write(FlowFile source, OutputStreamCallback writer) throws FlowFileAccessException;
+
+ /**
+ * Executes the given callback against the content corresponding to the
+ * given flow file.
+ *
+ * <i>Note</i>: The InputStream & OutputStream provided to the given
+ * StreamCallback will not be accessible once this method has completed its
+ * execution.
+ *
+ * @param source to read from and write to
+ * @param writer used to read the old content and write new content
+ * @return updated FlowFile
+ * @throws IllegalStateException if detected that this method is being
+ * called from within a callback of another method in this session and for
+ * the given FlowFile(s)
+ * @throws FlowFileHandlingException if the given FlowFile is already
+ * transferred or removed or doesn't belong to this session. Automatic
+ * rollback will occur.
+ * @throws MissingFlowFileException if the given FlowFile content cannot be
+ * found. The FlowFile should no longer be reference, will be internally
+ * destroyed, and the session is automatically rolled back and what is left
+ * of the FlowFile is destroyed.
+ * @throws FlowFileAccessException if some IO problem occurs accessing
+ * FlowFile content; if an attempt is made to access the InputStream or
+ * OutputStream provided to the given StreamCallback after this method
+ * completed its execution
+ */
+ FlowFile write(FlowFile source, StreamCallback writer) throws FlowFileAccessException;
+
+ /**
+ * Executes the given callback against the content corresponding to the
+ * given FlowFile, such that any data written to the OutputStream of the
+ * content will be appended to the end of FlowFile.
+ *
+ * <i>Note</i>: The OutputStream provided to the given OutputStreamCallback
+ * will not be accessible once this method has completed its execution.
+ *
+ * @param source the flowfile for which content should be appended
+ * @param writer used to write new bytes to the flowfile content
+ * @return the updated flowfile reference for the new content
+ * @throws FlowFileAccessException if an attempt is made to access the
+ * OutputStream provided to the given OutputStreamCallaback after this
+ * method completed its execution
+ */
+ FlowFile append(FlowFile source, OutputStreamCallback writer) throws FlowFileAccessException;
+
+ /**
+ * Writes to the given FlowFile all content from the given content path.
+ *
+ * @param source the file from which content will be obtained
+ * @param keepSourceFile if true the content is simply copied; if false the
+ * original content might be used in a destructive way for efficiency such
+ * that the repository will have the data but the original data will be
+ * gone. If false the source object will be removed or gone once imported.
+ * It will not be restored if the session is rolled back so this must be
+ * used with caution. In some cases it can result in tremendous efficiency
+ * gains but is also dangerous.
+ * @param destination the FlowFile whose content will be updated
+ * @return the updated destination FlowFile (new size)
+ * @throws IllegalStateException if detected that this method is being
+ * called from within a callback of another method in this session and for
+ * the given FlowFile(s)
+ * @throws FlowFileHandlingException if the given FlowFile is already
+ * transferred or removed or doesn't belong to this session. Automatic
+ * rollback will occur.
+ * @throws MissingFlowFileException if the given FlowFile content cannot be
+ * found. The FlowFile should no longer be reference, will be internally
+ * destroyed, and the session is automatically rolled back and what is left
+ * of the FlowFile is destroyed.
+ * @throws FlowFileAccessException if some IO problem occurs accessing
+ * FlowFile content
+ */
+ FlowFile importFrom(Path source, boolean keepSourceFile, FlowFile destination);
+
+ /**
+ * Writes to the given FlowFile all content from the given content path.
+ *
+ * @param source the file from which content will be obtained
+ * @param destination the FlowFile whose content will be updated
+ * @return the updated destination FlowFile (new size)
+ * @throws IllegalStateException if detected that this method is being
+ * called from within a callback of another method in this session and for
+ * the given FlowFile(s)
+ * @throws FlowFileHandlingException if the given FlowFile is already
+ * transferred or removed or doesn't belong to this session. Automatic
+ * rollback will occur.
+ * @throws MissingFlowFileException if the given FlowFile content cannot be
+ * found. The FlowFile should no longer be reference, will be internally
+ * destroyed, and the session is automatically rolled back and what is left
+ * of the FlowFile is destroyed.
+ * @throws FlowFileAccessException if some IO problem occurs accessing
+ * FlowFile content
+ */
+ FlowFile importFrom(InputStream source, FlowFile destination);
+
+ /**
+ * Writes the content of the given FlowFile to the given destination path.
+ *
+ * @param flowFile to export the content of
+ * @param destination to export the content to
+ * @param append if true will append to the current content at the given
+ * path; if false will replace any current content
+ * @throws IllegalStateException if detected that this method is being
+ * called from within a callback of another method in this session and for
+ * the given FlowFile(s)
+ * @throws FlowFileHandlingException if the given FlowFile is already
+ * transferred or removed or doesn't belong to this session. Automatic
+ * rollback will occur.
+ * @throws MissingFlowFileException if the given FlowFile content cannot be
+ * found. The FlowFile should no longer be reference, will be internally
+ * destroyed, and the session is automatically rolled back and what is left
+ * of the FlowFile is destroyed.
+ * @throws FlowFileAccessException if some IO problem occurs accessing
+ * FlowFile content
+ */
+ void exportTo(FlowFile flowFile, Path destination, boolean append);
+
+ /**
+ * Writes the content of the given FlowFile to the given destination stream
+ *
+ * @param flowFile to export the content of
+ * @param destination to export the content to
+ * @throws IllegalStateException if detected that this method is being
+ * called from within a callback of another method in this session and for
+ * the given FlowFile(s)
+ * @throws FlowFileHandlingException if the given FlowFile is already
+ * transferred or removed or doesn't belong to this session. Automatic
+ * rollback will occur.
+ * @throws MissingFlowFileException if the given FlowFile content cannot be
+ * found. The FlowFile should no longer be reference, will be internally
+ * destroyed, and the session is automatically rolled back and what is left
+ * of the FlowFile is destroyed.
+ * @throws FlowFileAccessException if some IO problem occurs accessing
+ * FlowFile content
+ */
+ void exportTo(FlowFile flowFile, OutputStream destination);
+
+ /**
+ * Returns a ProvenanceReporter that is tied to this ProcessSession.
+ *
+ * @return the provenance reporter
+ */
+ ProvenanceReporter getProvenanceReporter();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/processor/ProcessSessionFactory.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/ProcessSessionFactory.java b/nifi-api/src/main/java/org/apache/nifi/processor/ProcessSessionFactory.java
new file mode 100644
index 0000000..06c0479
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/processor/ProcessSessionFactory.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor;
+
+/**
+ * <p>
+ * Factory for creating a {@link ProcessSession}
+ * </p>
+ *
+ * <p>
+ * MUST BE THREAD-SAFE</p>
+ */
+public interface ProcessSessionFactory {
+
+ ProcessSession createSession();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/processor/Processor.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/Processor.java b/nifi-api/src/main/java/org/apache/nifi/processor/Processor.java
new file mode 100644
index 0000000..53c7c70
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/processor/Processor.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor;
+
+import java.util.Set;
+
+import org.apache.nifi.components.ConfigurableComponent;
+import org.apache.nifi.processor.exception.ProcessException;
+
+/**
+ * <p>
+ * Processor objects operate on FlowFile objects where the processors are linked
+ * together via relationships forming a directed graph structure.</p>
+ *
+ * <p>
+ * The validity of a processor once established is considered safe until a
+ * configuration property of that processor is changed. Even if the property
+ * change is itself valid it still brings the validity of the processor as a
+ * whole into question. Therefore changing a processor as a whole must be
+ * reverified. Also, changing any configuration property of a processor can also
+ * imply that its supported relationships have changed.</p>
+ *
+ * <p>
+ * Each Processor object is a single node in a flow graph. The same processor
+ * object on a graph may be called concurrently to update its configuration
+ * state and to perform 'onTrigger' actions. The framework also provides
+ * numerous hooks via annotations that subclasses can use to control the
+ * lifecycle of a given processor instance.</p>
+ *
+ * <p>
+ * Processor objects are expected to be thread-safe and must have a public
+ * default no-args constructor to facilitate the java service loader
+ * mechanism.</p>
+ *
+ */
+public interface Processor extends ConfigurableComponent {
+
+ /**
+ * Provides the processor with access to objects that may be of use
+ * throughout the life of the Processor
+ *
+ * @param context of initialization
+ */
+ void initialize(ProcessorInitializationContext context);
+
+ /**
+ * @return Set of all relationships this processor expects to transfer a
+ * flow file to. An empty set indicates this processor does not have any
+ * destination relationships. Guaranteed non null.
+ */
+ Set<Relationship> getRelationships();
+
+ /**
+ * <p>
+ * The method called when this processor is triggered to operate by the
+ * controller. This method may be called concurrently from different
+ * threads. When this method is called depends on how this processor is
+ * configured within a controller to be triggered (timing or event
+ * based).</p>
+ *
+ * @param context provides access to convenience methods for obtaining
+ * property values, delaying the scheduling of the processor, provides
+ * access to Controller Services, etc.
+ * @param sessionFactory provides access to a {@link ProcessSession}, which
+ * can be used for accessing FlowFiles, etc.
+ *
+ * @throws ProcessException if processing did not complete normally though
+ * indicates the problem is an understood potential outcome of processing.
+ * The controller/caller will handle these exceptions gracefully such as
+ * logging, etc.. If another type of exception is allowed to propagate the
+ * controller may no longer trigger this processor to operate, as this would
+ * indicate a probable coding defect.
+ */
+ void onTrigger(ProcessContext context, ProcessSessionFactory sessionFactory) throws ProcessException;
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/processor/ProcessorInitializationContext.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/ProcessorInitializationContext.java b/nifi-api/src/main/java/org/apache/nifi/processor/ProcessorInitializationContext.java
new file mode 100644
index 0000000..7b09e1b
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/processor/ProcessorInitializationContext.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor;
+
+import org.apache.nifi.controller.ControllerServiceLookup;
+import org.apache.nifi.logging.ProcessorLog;
+
+/**
+ * <p>
+ * The <code>ProcessorInitializationContext</code> provides
+ * {@link nifi.processor.Processor Processor}s access to objects that may be of
+ * use throughout the life of the Processor.
+ * </p>
+ */
+public interface ProcessorInitializationContext {
+
+ /**
+ * @return the unique identifier for this processor
+ */
+ String getIdentifier();
+
+ /**
+ * @return a {@link ProcessorLog} that is tied to this processor that can be
+ * used to log events
+ */
+ ProcessorLog getLogger();
+
+ /**
+ * @return the {@link ControllerServiceLookup} which can be used to obtain
+ * Controller Services
+ */
+ ControllerServiceLookup getControllerServiceLookup();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/processor/QueueSize.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/QueueSize.java b/nifi-api/src/main/java/org/apache/nifi/processor/QueueSize.java
new file mode 100644
index 0000000..c3c2ccc
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/processor/QueueSize.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor;
+
+/**
+ *
+ */
+public class QueueSize {
+
+ private final int objectCount;
+ private final long totalSizeBytes;
+
+ public QueueSize(final int numberObjects, final long totalSizeBytes) {
+ if (numberObjects < 0 || totalSizeBytes < 0) {
+ throw new IllegalArgumentException();
+ }
+ objectCount = numberObjects;
+ this.totalSizeBytes = totalSizeBytes;
+ }
+
+ /**
+ * @return number of objects present on the queue
+ */
+ public int getObjectCount() {
+ return objectCount;
+ }
+
+ /**
+ * @return total size in bytes of the content for the data on the queue
+ */
+ public long getByteCount() {
+ return totalSizeBytes;
+ }
+}
[23/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/DataOutputStream.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/DataOutputStream.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/DataOutputStream.java
new file mode 100644
index 0000000..e205996
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/DataOutputStream.java
@@ -0,0 +1,370 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io;
+
+import java.io.DataOutput;
+import java.io.FilterOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.UTFDataFormatException;
+
+/**
+ * This class is different from java.io.DataOutputStream in that it does synchronize on its methods.
+ */
+public class DataOutputStream extends FilterOutputStream implements DataOutput {
+
+ /**
+ * The number of bytes written to the data output stream so far. If this counter overflows, it will be wrapped to Integer.MAX_VALUE.
+ */
+ protected int written;
+
+ /**
+ * bytearr is initialized on demand by writeUTF
+ */
+ private byte[] bytearr = null;
+
+ /**
+ * Creates a new data output stream to write data to the specified underlying output stream. The counter <code>written</code> is set to zero.
+ *
+ * @param out the underlying output stream, to be saved for later use.
+ * @see java.io.FilterOutputStream#out
+ */
+ public DataOutputStream(OutputStream out) {
+ super(out);
+ }
+
+ /**
+ * Increases the written counter by the specified value until it reaches Integer.MAX_VALUE.
+ */
+ private void incCount(int value) {
+ int temp = written + value;
+ if (temp < 0) {
+ temp = Integer.MAX_VALUE;
+ }
+ written = temp;
+ }
+
+ /**
+ * Writes the specified byte (the low eight bits of the argument <code>b</code>) to the underlying output stream. If no exception is thrown, the counter <code>written</code> is incremented by
+ * <code>1</code>.
+ * <p>
+ * Implements the <code>write</code> method of <code>OutputStream</code>.
+ *
+ * @param b the <code>byte</code> to be written.
+ * @exception IOException if an I/O error occurs.
+ * @see java.io.FilterOutputStream#out
+ */
+ @Override
+ public void write(int b) throws IOException {
+ out.write(b);
+ incCount(1);
+ }
+
+ /**
+ * Writes <code>len</code> bytes from the specified byte array starting at offset <code>off</code> to the underlying output stream. If no exception is thrown, the counter <code>written</code> is
+ * incremented by <code>len</code>.
+ *
+ * @param b the data.
+ * @param off the start offset in the data.
+ * @param len the number of bytes to write.
+ * @exception IOException if an I/O error occurs.
+ * @see java.io.FilterOutputStream#out
+ */
+ @Override
+ public void write(byte b[], int off, int len) throws IOException {
+ out.write(b, off, len);
+ incCount(len);
+ }
+
+ /**
+ * Flushes this data output stream. This forces any buffered output bytes to be written out to the stream.
+ * <p>
+ * The <code>flush</code> method of <code>DataOutputStream</code> calls the <code>flush</code> method of its underlying output stream.
+ *
+ * @exception IOException if an I/O error occurs.
+ * @see java.io.FilterOutputStream#out
+ * @see java.io.OutputStream#flush()
+ */
+ @Override
+ public void flush() throws IOException {
+ out.flush();
+ }
+
+ /**
+ * Writes a <code>boolean</code> to the underlying output stream as a 1-byte value. The value <code>true</code> is written out as the value <code>(byte)1</code>; the value <code>false</code> is
+ * written out as the value <code>(byte)0</code>. If no exception is thrown, the counter <code>written</code> is incremented by <code>1</code>.
+ *
+ * @param v a <code>boolean</code> value to be written.
+ * @exception IOException if an I/O error occurs.
+ * @see java.io.FilterOutputStream#out
+ */
+ @Override
+ public final void writeBoolean(boolean v) throws IOException {
+ out.write(v ? 1 : 0);
+ incCount(1);
+ }
+
+ /**
+ * Writes out a <code>byte</code> to the underlying output stream as a 1-byte value. If no exception is thrown, the counter <code>written</code> is incremented by <code>1</code>.
+ *
+ * @param v a <code>byte</code> value to be written.
+ * @exception IOException if an I/O error occurs.
+ * @see java.io.FilterOutputStream#out
+ */
+ @Override
+ public final void writeByte(int v) throws IOException {
+ out.write(v);
+ incCount(1);
+ }
+
+ /**
+ * Writes a <code>short</code> to the underlying output stream as two bytes, high byte first. If no exception is thrown, the counter <code>written</code> is incremented by <code>2</code>.
+ *
+ * @param v a <code>short</code> to be written.
+ * @exception IOException if an I/O error occurs.
+ * @see java.io.FilterOutputStream#out
+ */
+ @Override
+ public final void writeShort(int v) throws IOException {
+ out.write((v >>> 8) & 0xFF);
+ out.write((v) & 0xFF);
+ incCount(2);
+ }
+
+ /**
+ * Writes a <code>char</code> to the underlying output stream as a 2-byte value, high byte first. If no exception is thrown, the counter <code>written</code> is incremented by <code>2</code>.
+ *
+ * @param v a <code>char</code> value to be written.
+ * @exception IOException if an I/O error occurs.
+ * @see java.io.FilterOutputStream#out
+ */
+ @Override
+ public final void writeChar(int v) throws IOException {
+ out.write((v >>> 8) & 0xFF);
+ out.write((v) & 0xFF);
+ incCount(2);
+ }
+
+ /**
+ * Writes an <code>int</code> to the underlying output stream as four bytes, high byte first. If no exception is thrown, the counter <code>written</code> is incremented by <code>4</code>.
+ *
+ * @param v an <code>int</code> to be written.
+ * @exception IOException if an I/O error occurs.
+ * @see java.io.FilterOutputStream#out
+ */
+ @Override
+ public final void writeInt(int v) throws IOException {
+ out.write((v >>> 24) & 0xFF);
+ out.write((v >>> 16) & 0xFF);
+ out.write((v >>> 8) & 0xFF);
+ out.write((v) & 0xFF);
+ incCount(4);
+ }
+
+ private final byte writeBuffer[] = new byte[8];
+
+ /**
+ * Writes a <code>long</code> to the underlying output stream as eight bytes, high byte first. In no exception is thrown, the counter <code>written</code> is incremented by <code>8</code>.
+ *
+ * @param v a <code>long</code> to be written.
+ * @exception IOException if an I/O error occurs.
+ * @see java.io.FilterOutputStream#out
+ */
+ @Override
+ public final void writeLong(long v) throws IOException {
+ writeBuffer[0] = (byte) (v >>> 56);
+ writeBuffer[1] = (byte) (v >>> 48);
+ writeBuffer[2] = (byte) (v >>> 40);
+ writeBuffer[3] = (byte) (v >>> 32);
+ writeBuffer[4] = (byte) (v >>> 24);
+ writeBuffer[5] = (byte) (v >>> 16);
+ writeBuffer[6] = (byte) (v >>> 8);
+ writeBuffer[7] = (byte) (v);
+ out.write(writeBuffer, 0, 8);
+ incCount(8);
+ }
+
+ /**
+ * Converts the float argument to an <code>int</code> using the <code>floatToIntBits</code> method in class <code>Float</code>, and then writes that <code>int</code> value to the underlying output
+ * stream as a 4-byte quantity, high byte first. If no exception is thrown, the counter <code>written</code> is incremented by <code>4</code>.
+ *
+ * @param v a <code>float</code> value to be written.
+ * @exception IOException if an I/O error occurs.
+ * @see java.io.FilterOutputStream#out
+ * @see java.lang.Float#floatToIntBits(float)
+ */
+ @Override
+ public final void writeFloat(float v) throws IOException {
+ writeInt(Float.floatToIntBits(v));
+ }
+
+ /**
+ * Converts the double argument to a <code>long</code> using the <code>doubleToLongBits</code> method in class <code>Double</code>, and then writes that <code>long</code> value to the underlying
+ * output stream as an 8-byte quantity, high byte first. If no exception is thrown, the counter <code>written</code> is incremented by <code>8</code>.
+ *
+ * @param v a <code>double</code> value to be written.
+ * @exception IOException if an I/O error occurs.
+ * @see java.io.FilterOutputStream#out
+ * @see java.lang.Double#doubleToLongBits(double)
+ */
+ @Override
+ public final void writeDouble(double v) throws IOException {
+ writeLong(Double.doubleToLongBits(v));
+ }
+
+ /**
+ * Writes out the string to the underlying output stream as a sequence of bytes. Each character in the string is written out, in sequence, by discarding its high eight bits. If no exception is
+ * thrown, the counter <code>written</code> is incremented by the length of <code>s</code>.
+ *
+ * @param s a string of bytes to be written.
+ * @exception IOException if an I/O error occurs.
+ * @see java.io.FilterOutputStream#out
+ */
+ @Override
+ public final void writeBytes(String s) throws IOException {
+ int len = s.length();
+ for (int i = 0; i < len; i++) {
+ out.write((byte) s.charAt(i));
+ }
+ incCount(len);
+ }
+
+ /**
+ * Writes a string to the underlying output stream as a sequence of characters. Each character is written to the data output stream as if by the <code>writeChar</code> method. If no exception is
+ * thrown, the counter <code>written</code> is incremented by twice the length of <code>s</code>.
+ *
+ * @param s a <code>String</code> value to be written.
+ * @exception IOException if an I/O error occurs.
+ * @see java.io.DataOutputStream#writeChar(int)
+ * @see java.io.FilterOutputStream#out
+ */
+ @Override
+ public final void writeChars(String s) throws IOException {
+ int len = s.length();
+ for (int i = 0; i < len; i++) {
+ int v = s.charAt(i);
+ out.write((v >>> 8) & 0xFF);
+ out.write((v) & 0xFF);
+ }
+ incCount(len * 2);
+ }
+
+ /**
+ * Writes a string to the underlying output stream using
+ * <a href="DataInput.html#modified-utf-8">modified UTF-8</a>
+ * encoding in a machine-independent manner.
+ * <p>
+ * First, two bytes are written to the output stream as if by the <code>writeShort</code> method giving the number of bytes to follow. This value is the number of bytes actually written out, not
+ * the length of the string. Following the length, each character of the string is output, in sequence, using the modified UTF-8 encoding for the character. If no exception is thrown, the counter
+ * <code>written</code> is incremented by the total number of bytes written to the output stream. This will be at least two plus the length of <code>str</code>, and at most two plus thrice the
+ * length of <code>str</code>.
+ *
+ * @param str a string to be written.
+ * @exception IOException if an I/O error occurs.
+ */
+ @Override
+ public final void writeUTF(String str) throws IOException {
+ writeUTF(str, this);
+ }
+
+ /**
+ * Writes a string to the specified DataOutput using
+ * <a href="DataInput.html#modified-utf-8">modified UTF-8</a>
+ * encoding in a machine-independent manner.
+ * <p>
+ * First, two bytes are written to out as if by the <code>writeShort</code> method giving the number of bytes to follow. This value is the number of bytes actually written out, not the length of
+ * the string. Following the length, each character of the string is output, in sequence, using the modified UTF-8 encoding for the character. If no exception is thrown, the counter
+ * <code>written</code> is incremented by the total number of bytes written to the output stream. This will be at least two plus the length of <code>str</code>, and at most two plus thrice the
+ * length of <code>str</code>.
+ *
+ * @param str a string to be written.
+ * @param out destination to write to
+ * @return The number of bytes written out.
+ * @exception IOException if an I/O error occurs.
+ */
+ static int writeUTF(String str, DataOutput out) throws IOException {
+ int strlen = str.length();
+ int utflen = 0;
+ int c, count = 0;
+
+ /* use charAt instead of copying String to char array */
+ for (int i = 0; i < strlen; i++) {
+ c = str.charAt(i);
+ if ((c >= 0x0001) && (c <= 0x007F)) {
+ utflen++;
+ } else if (c > 0x07FF) {
+ utflen += 3;
+ } else {
+ utflen += 2;
+ }
+ }
+
+ if (utflen > 65535) {
+ throw new UTFDataFormatException("encoded string too long: " + utflen + " bytes");
+ }
+
+ byte[] bytearr = null;
+ if (out instanceof DataOutputStream) {
+ DataOutputStream dos = (DataOutputStream) out;
+ if (dos.bytearr == null || (dos.bytearr.length < (utflen + 2))) {
+ dos.bytearr = new byte[(utflen * 2) + 2];
+ }
+ bytearr = dos.bytearr;
+ } else {
+ bytearr = new byte[utflen + 2];
+ }
+
+ bytearr[count++] = (byte) ((utflen >>> 8) & 0xFF);
+ bytearr[count++] = (byte) ((utflen) & 0xFF);
+
+ int i = 0;
+ for (i = 0; i < strlen; i++) {
+ c = str.charAt(i);
+ if (!((c >= 0x0001) && (c <= 0x007F))) {
+ break;
+ }
+ bytearr[count++] = (byte) c;
+ }
+
+ for (; i < strlen; i++) {
+ c = str.charAt(i);
+ if ((c >= 0x0001) && (c <= 0x007F)) {
+ bytearr[count++] = (byte) c;
+
+ } else if (c > 0x07FF) {
+ bytearr[count++] = (byte) (0xE0 | ((c >> 12) & 0x0F));
+ bytearr[count++] = (byte) (0x80 | ((c >> 6) & 0x3F));
+ bytearr[count++] = (byte) (0x80 | ((c) & 0x3F));
+ } else {
+ bytearr[count++] = (byte) (0xC0 | ((c >> 6) & 0x1F));
+ bytearr[count++] = (byte) (0x80 | ((c) & 0x3F));
+ }
+ }
+ out.write(bytearr, 0, utflen + 2);
+ return utflen + 2;
+ }
+
+ /**
+ * Returns the current value of the counter <code>written</code>, the number of bytes written to this data output stream so far. If the counter overflows, it will be wrapped to Integer.MAX_VALUE.
+ *
+ * @return the value of the <code>written</code> field.
+ * @see java.io.DataOutputStream#written
+ */
+ public final int size() {
+ return written;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/GZIPOutputStream.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/GZIPOutputStream.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/GZIPOutputStream.java
new file mode 100644
index 0000000..1e2f3c7
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/GZIPOutputStream.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * <p>
+ * This class extends the {@link java.util.zip.GZIPOutputStream} by allowing the constructor to provide a compression level, and uses a default value of 1, rather than 5.
+ * </p>
+ */
+public class GZIPOutputStream extends java.util.zip.GZIPOutputStream {
+
+ public static final int DEFAULT_COMPRESSION_LEVEL = 1;
+
+ public GZIPOutputStream(final OutputStream out) throws IOException {
+ this(out, DEFAULT_COMPRESSION_LEVEL);
+ }
+
+ public GZIPOutputStream(final OutputStream out, final int compressionLevel) throws IOException {
+ super(out);
+ def.setLevel(compressionLevel);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/LeakyBucketStreamThrottler.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/LeakyBucketStreamThrottler.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/LeakyBucketStreamThrottler.java
new file mode 100644
index 0000000..5153db5
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/LeakyBucketStreamThrottler.java
@@ -0,0 +1,330 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class LeakyBucketStreamThrottler implements StreamThrottler {
+
+ private final int maxBytesPerSecond;
+ private final BlockingQueue<Request> requestQueue = new LinkedBlockingQueue<Request>();
+ private final ScheduledExecutorService executorService;
+ private final AtomicBoolean shutdown = new AtomicBoolean(false);
+
+ public LeakyBucketStreamThrottler(final int maxBytesPerSecond) {
+ this.maxBytesPerSecond = maxBytesPerSecond;
+
+ executorService = Executors.newSingleThreadScheduledExecutor();
+ final Runnable task = new Drain();
+ executorService.scheduleAtFixedRate(task, 0, 1000, TimeUnit.MILLISECONDS);
+ }
+
+ @Override
+ public void close() {
+ this.shutdown.set(true);
+
+ executorService.shutdown();
+ try {
+ // Should not take more than 2 seconds because we run every second. If it takes more than
+ // 2 seconds, it is because the Runnable thread is blocking on a write; in this case,
+ // we will just ignore it and return
+ executorService.awaitTermination(2, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ }
+ }
+
+ @Override
+ public OutputStream newThrottledOutputStream(final OutputStream toWrap) {
+ return new OutputStream() {
+ @Override
+ public void write(final int b) throws IOException {
+ write(new byte[]{(byte) b}, 0, 1);
+ }
+
+ @Override
+ public void write(byte[] b) throws IOException {
+ write(b, 0, b.length);
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) throws IOException {
+ final InputStream in = new ByteArrayInputStream(b, off, len);
+ LeakyBucketStreamThrottler.this.copy(in, toWrap);
+ }
+
+ @Override
+ public void close() throws IOException {
+ toWrap.close();
+ }
+
+ @Override
+ public void flush() throws IOException {
+ toWrap.flush();
+ }
+ };
+ }
+
+ @Override
+ public InputStream newThrottledInputStream(final InputStream toWrap) {
+ return new InputStream() {
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ @Override
+ public int read() throws IOException {
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream(1);
+ LeakyBucketStreamThrottler.this.copy(toWrap, baos, 1L);
+ if (baos.getBufferLength() < 1) {
+ return -1;
+ }
+
+ return baos.getUnderlyingBuffer()[0] & 0xFF;
+ }
+
+ @Override
+ public int read(final byte[] b) throws IOException {
+ if (b.length == 0) {
+ return 0;
+ }
+ return read(b, 0, b.length);
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ if (len < 0) {
+ throw new IllegalArgumentException();
+ }
+ if (len == 0) {
+ return 0;
+ }
+
+ baos.reset();
+ final int copied = (int) LeakyBucketStreamThrottler.this.copy(toWrap, baos, len);
+ if (copied == 0) {
+ return -1;
+ }
+ System.arraycopy(baos.getUnderlyingBuffer(), 0, b, off, copied);
+ return copied;
+ }
+
+ @Override
+ public void close() throws IOException {
+ toWrap.close();
+ }
+
+ @Override
+ public int available() throws IOException {
+ return toWrap.available();
+ }
+ };
+ }
+
+ @Override
+ public long copy(final InputStream in, final OutputStream out) throws IOException {
+ return copy(in, out, -1);
+ }
+
+ @Override
+ public long copy(final InputStream in, final OutputStream out, final long maxBytes) throws IOException {
+ long totalBytesCopied = 0;
+ boolean finished = false;
+ while (!finished) {
+ final long requestMax = (maxBytes < 0) ? Long.MAX_VALUE : maxBytes - totalBytesCopied;
+ final Request request = new Request(in, out, requestMax);
+ boolean transferred = false;
+ while (!transferred) {
+ if (shutdown.get()) {
+ throw new IOException("Throttler shutdown");
+ }
+
+ try {
+ transferred = requestQueue.offer(request, 1000, TimeUnit.MILLISECONDS);
+ } catch (final InterruptedException e) {
+ throw new IOException("Interrupted", e);
+ }
+ }
+
+ final BlockingQueue<Response> responseQueue = request.getResponseQueue();
+ Response response = null;
+ while (response == null) {
+ try {
+ if (shutdown.get()) {
+ throw new IOException("Throttler shutdown");
+ }
+ response = responseQueue.poll(1000L, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ throw new IOException("Interrupted", e);
+ }
+ }
+
+ if (!response.isSuccess()) {
+ throw response.getError();
+ }
+
+ totalBytesCopied += response.getBytesCopied();
+ finished = (response.getBytesCopied() == 0) || (totalBytesCopied >= maxBytes && maxBytes > 0);
+ }
+
+ return totalBytesCopied;
+ }
+
+ /**
+ * This class is responsible for draining water from the leaky bucket. I.e., it actually moves the data
+ */
+ private class Drain implements Runnable {
+
+ private final byte[] buffer;
+
+ public Drain() {
+ final int bufferSize = Math.min(4096, maxBytesPerSecond);
+ buffer = new byte[bufferSize];
+ }
+
+ @Override
+ public void run() {
+ final long start = System.currentTimeMillis();
+
+ int bytesTransferred = 0;
+ while (bytesTransferred < maxBytesPerSecond) {
+ final long maxMillisToWait = 1000 - (System.currentTimeMillis() - start);
+ if (maxMillisToWait < 1) {
+ return;
+ }
+
+ try {
+ final Request request = requestQueue.poll(maxMillisToWait, TimeUnit.MILLISECONDS);
+ if (request == null) {
+ return;
+ }
+
+ final BlockingQueue<Response> responseQueue = request.getResponseQueue();
+
+ final OutputStream out = request.getOutputStream();
+ final InputStream in = request.getInputStream();
+
+ try {
+ final long requestMax = request.getMaxBytesToCopy();
+ long maxBytesToTransfer;
+ if (requestMax < 0) {
+ maxBytesToTransfer = Math.min(buffer.length, maxBytesPerSecond - bytesTransferred);
+ } else {
+ maxBytesToTransfer = Math.min(requestMax,
+ Math.min(buffer.length, maxBytesPerSecond - bytesTransferred));
+ }
+ maxBytesToTransfer = Math.max(1L, maxBytesToTransfer);
+
+ final int bytesCopied = fillBuffer(in, maxBytesToTransfer);
+ out.write(buffer, 0, bytesCopied);
+
+ final Response response = new Response(true, bytesCopied);
+ responseQueue.put(response);
+ bytesTransferred += bytesCopied;
+ } catch (final IOException e) {
+ final Response response = new Response(e);
+ responseQueue.put(response);
+ }
+ } catch (InterruptedException e) {
+ }
+ }
+ }
+
+ private int fillBuffer(final InputStream in, final long maxBytes) throws IOException {
+ int bytesRead = 0;
+ int len;
+ while (bytesRead < maxBytes && (len = in.read(buffer, bytesRead, (int) Math.min(maxBytes - bytesRead, buffer.length - bytesRead))) > 0) {
+ bytesRead += len;
+ }
+
+ return bytesRead;
+ }
+ }
+
+ private static class Response {
+
+ private final boolean success;
+ private final IOException error;
+ private final int bytesCopied;
+
+ public Response(final boolean success, final int bytesCopied) {
+ this.success = success;
+ this.bytesCopied = bytesCopied;
+ this.error = null;
+ }
+
+ public Response(final IOException error) {
+ this.success = false;
+ this.error = error;
+ this.bytesCopied = -1;
+ }
+
+ public boolean isSuccess() {
+ return success;
+ }
+
+ public IOException getError() {
+ return error;
+ }
+
+ public int getBytesCopied() {
+ return bytesCopied;
+ }
+ }
+
+ private static class Request {
+
+ private final OutputStream out;
+ private final InputStream in;
+ private final long maxBytesToCopy;
+ private final BlockingQueue<Response> responseQueue;
+
+ public Request(final InputStream in, final OutputStream out, final long maxBytesToCopy) {
+ this.out = out;
+ this.in = in;
+ this.maxBytesToCopy = maxBytesToCopy;
+ this.responseQueue = new LinkedBlockingQueue<Response>(1);
+ }
+
+ public BlockingQueue<Response> getResponseQueue() {
+ return this.responseQueue;
+ }
+
+ public OutputStream getOutputStream() {
+ return out;
+ }
+
+ public InputStream getInputStream() {
+ return in;
+ }
+
+ public long getMaxBytesToCopy() {
+ return maxBytesToCopy;
+ }
+
+ @Override
+ public String toString() {
+ return "Request[maxBytes=" + maxBytesToCopy + "]";
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/LimitingInputStream.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/LimitingInputStream.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/LimitingInputStream.java
new file mode 100644
index 0000000..a657030
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/LimitingInputStream.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+public class LimitingInputStream extends InputStream {
+
+ private final InputStream in;
+ private final long limit;
+ private long bytesRead = 0;
+
+ public LimitingInputStream(final InputStream in, final long limit) {
+ this.in = in;
+ this.limit = limit;
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (bytesRead >= limit) {
+ return -1;
+ }
+
+ final int val = in.read();
+ if (val > -1) {
+ bytesRead++;
+ }
+ return val;
+ }
+
+ @Override
+ public int read(final byte[] b) throws IOException {
+ if (bytesRead >= limit) {
+ return -1;
+ }
+
+ final int maxToRead = (int) Math.min(b.length, limit - bytesRead);
+
+ final int val = in.read(b, 0, maxToRead);
+ if (val > 0) {
+ bytesRead += val;
+ }
+ return val;
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ if (bytesRead >= limit) {
+ return -1;
+ }
+
+ final int maxToRead = (int) Math.min(len, limit - bytesRead);
+
+ final int val = in.read(b, off, maxToRead);
+ if (val > 0) {
+ bytesRead += val;
+ }
+ return val;
+ }
+
+ @Override
+ public long skip(final long n) throws IOException {
+ final long skipped = in.skip(Math.min(n, limit - bytesRead));
+ bytesRead += skipped;
+ return skipped;
+ }
+
+ @Override
+ public int available() throws IOException {
+ return in.available();
+ }
+
+ @Override
+ public void close() throws IOException {
+ in.close();
+ }
+
+ @Override
+ public void mark(int readlimit) {
+ in.mark(readlimit);
+ }
+
+ @Override
+ public boolean markSupported() {
+ return in.markSupported();
+ }
+
+ @Override
+ public void reset() throws IOException {
+ in.reset();
+ }
+
+ public long getLimit() {
+ return limit;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/MinimumLengthInputStream.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/MinimumLengthInputStream.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/MinimumLengthInputStream.java
new file mode 100644
index 0000000..44e9c2e
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/MinimumLengthInputStream.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io;
+
+import java.io.EOFException;
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * An InputStream that will throw EOFException if the underlying InputStream runs out of data before reaching the configured minimum amount of data
+ */
+public class MinimumLengthInputStream extends FilterInputStream {
+
+ private final long minLength;
+ private long consumedCount = 0L;
+
+ public MinimumLengthInputStream(final InputStream in, final long minLength) {
+ super(in);
+ this.minLength = minLength;
+ }
+
+ @Override
+ public int read() throws IOException {
+ final int b = super.read();
+ if (b < 0 && consumedCount < minLength) {
+ throw new EOFException();
+ }
+
+ if (b >= 0) {
+ consumedCount++;
+ }
+
+ return b;
+ }
+
+ @Override
+ public int read(byte[] b) throws IOException {
+ return read(b, 0, b.length);
+ }
+
+ public int read(byte[] b, int off, int len) throws IOException {
+ final int num = super.read(b, off, len);
+
+ if (num < 0 && consumedCount < minLength) {
+ throw new EOFException();
+ }
+
+ if (num >= 0) {
+ consumedCount += num;
+ }
+
+ return num;
+ }
+
+ @Override
+ public long skip(final long n) throws IOException {
+ long skipped = super.skip(n);
+ if (skipped < 1) {
+ final int b = super.read();
+ if (b >= 0) {
+ skipped = 1;
+ }
+ }
+
+ if (skipped < 0 && consumedCount < minLength) {
+ throw new EOFException();
+ }
+
+ if (skipped >= 0) {
+ consumedCount += skipped;
+ }
+
+ return skipped;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/NonCloseableInputStream.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/NonCloseableInputStream.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/NonCloseableInputStream.java
new file mode 100644
index 0000000..27a0c47
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/NonCloseableInputStream.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io;
+
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * Wraps and InputStream so that the underlying InputStream cannot be closed. This is used so that the InputStream can be wrapped with yet another InputStream and prevent the outer layer from closing
+ * the inner InputStream
+ */
+public class NonCloseableInputStream extends FilterInputStream {
+
+ private final InputStream toWrap;
+
+ public NonCloseableInputStream(final InputStream toWrap) {
+ super(toWrap);
+ this.toWrap = toWrap;
+ }
+
+ @Override
+ public int read() throws IOException {
+ return toWrap.read();
+ }
+
+ @Override
+ public int read(byte[] b) throws IOException {
+ return toWrap.read(b);
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ return toWrap.read(b, off, len);
+ }
+
+ @Override
+ public void close() throws IOException {
+ // do nothing
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/NonCloseableOutputStream.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/NonCloseableOutputStream.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/NonCloseableOutputStream.java
new file mode 100644
index 0000000..9c77637
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/NonCloseableOutputStream.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io;
+
+import java.io.FilterOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+public class NonCloseableOutputStream extends FilterOutputStream {
+
+ private final OutputStream out;
+
+ public NonCloseableOutputStream(final OutputStream out) {
+ super(out);
+ this.out = out;
+ }
+
+ @Override
+ public void write(byte[] b) throws IOException {
+ out.write(b);
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) throws IOException {
+ out.write(b, off, len);
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ out.write(b);
+ }
+
+ @Override
+ public void close() throws IOException {
+ out.flush();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/NullOutputStream.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/NullOutputStream.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/NullOutputStream.java
new file mode 100644
index 0000000..8452761
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/NullOutputStream.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * OutputStream that throws away all data, just like as if writing to /dev/null
+ */
+public class NullOutputStream extends OutputStream {
+
+ @Override
+ public void write(final int b) throws IOException {
+ }
+
+ @Override
+ public void write(final byte[] b) throws IOException {
+ }
+
+ @Override
+ public void write(final byte[] b, int off, int len) throws IOException {
+ }
+
+ @Override
+ public void close() throws IOException {
+ }
+
+ @Override
+ public void flush() throws IOException {
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/StreamThrottler.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/StreamThrottler.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/StreamThrottler.java
new file mode 100644
index 0000000..9158050
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/StreamThrottler.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+public interface StreamThrottler extends Closeable {
+
+ long copy(InputStream in, OutputStream out) throws IOException;
+
+ long copy(InputStream in, OutputStream out, long maxBytes) throws IOException;
+
+ InputStream newThrottledInputStream(final InputStream toWrap);
+
+ OutputStream newThrottledOutputStream(final OutputStream toWrap);
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/StreamUtils.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/StreamUtils.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/StreamUtils.java
new file mode 100644
index 0000000..64f6eaa
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/StreamUtils.java
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.nifi.stream.io.exception.BytePatternNotFoundException;
+import org.apache.nifi.stream.io.util.NonThreadSafeCircularBuffer;
+
+public class StreamUtils {
+
+ public static long copy(final InputStream source, final OutputStream destination) throws IOException {
+ final byte[] buffer = new byte[8192];
+ int len;
+ long totalCount = 0L;
+ while ((len = source.read(buffer)) > 0) {
+ destination.write(buffer, 0, len);
+ totalCount += len;
+ }
+ return totalCount;
+ }
+
+ /**
+ * Copies <code>numBytes</code> from <code>source</code> to <code>destination</code>. If <code>numBytes</code> are not available from <code>source</code>, throws EOFException
+ *
+ * @param source the source of bytes to copy
+ * @param destination the destination to copy bytes to
+ * @param numBytes the number of bytes to copy
+ * @throws IOException if any issues occur while copying
+ */
+ public static void copy(final InputStream source, final OutputStream destination, final long numBytes) throws IOException {
+ final byte[] buffer = new byte[8192];
+ int len;
+ long bytesLeft = numBytes;
+ while ((len = source.read(buffer, 0, (int) Math.min(bytesLeft, buffer.length))) > 0) {
+ destination.write(buffer, 0, len);
+ bytesLeft -= len;
+ }
+
+ if (bytesLeft > 0) {
+ throw new EOFException("Attempted to copy " + numBytes + " bytes but only " + (numBytes - bytesLeft) + " bytes were available");
+ }
+ }
+
+ /**
+ * Reads data from the given input stream, copying it to the destination byte array. If the InputStream has less data than the given byte array, throws an EOFException
+ *
+ * @param source the source to copy bytes from
+ * @param destination the destination to fill
+ * @throws IOException if any issues occur reading bytes
+ */
+ public static void fillBuffer(final InputStream source, final byte[] destination) throws IOException {
+ fillBuffer(source, destination, true);
+ }
+
+ /**
+ * Reads data from the given input stream, copying it to the destination byte array. If the InputStream has less data than the given byte array, throws an EOFException if
+ * <code>ensureCapacity</code> is true and otherwise returns the number of bytes copied
+ *
+ * @param source the source to read bytes from
+ * @param destination the destination to fill
+ * @param ensureCapacity whether or not to enforce that the InputStream have at least as much data as the capacity of the destination byte array
+ * @return the number of bytes actually filled
+ * @throws IOException if unable to read from the underlying stream
+ */
+ public static int fillBuffer(final InputStream source, final byte[] destination, final boolean ensureCapacity) throws IOException {
+ int bytesRead = 0;
+ int len;
+ while (bytesRead < destination.length) {
+ len = source.read(destination, bytesRead, destination.length - bytesRead);
+ if (len < 0) {
+ if (ensureCapacity) {
+ throw new EOFException();
+ } else {
+ break;
+ }
+ }
+
+ bytesRead += len;
+ }
+
+ return bytesRead;
+ }
+
+ /**
+ * Copies data from in to out until either we are out of data (returns null) or we hit one of the byte patterns identified by the <code>stoppers</code> parameter (returns the byte pattern
+ * matched). The bytes in the stopper will be copied.
+ *
+ * @param in the source to read bytes from
+ * @param out the destination to write bytes to
+ * @param maxBytes the max bytes to copy
+ * @param stoppers patterns of bytes which if seen will cause the copy to stop
+ * @return the byte array matched, or null if end of stream was reached
+ * @throws IOException if issues occur reading or writing bytes to the underlying streams
+ */
+ public static byte[] copyInclusive(final InputStream in, final OutputStream out, final int maxBytes, final byte[]... stoppers) throws IOException {
+ if (stoppers.length == 0) {
+ return null;
+ }
+
+ final List<NonThreadSafeCircularBuffer> circularBuffers = new ArrayList<>();
+ for (final byte[] stopper : stoppers) {
+ circularBuffers.add(new NonThreadSafeCircularBuffer(stopper));
+ }
+
+ long bytesRead = 0;
+ while (true) {
+ final int next = in.read();
+ if (next == -1) {
+ return null;
+ } else if (maxBytes > 0 && ++bytesRead >= maxBytes) {
+ throw new BytePatternNotFoundException("Did not encounter any byte pattern that was expected; data does not appear to be in the expected format");
+ }
+
+ out.write(next);
+
+ for (final NonThreadSafeCircularBuffer circ : circularBuffers) {
+ if (circ.addAndCompare((byte) next)) {
+ return circ.getByteArray();
+ }
+ }
+ }
+ }
+
+ /**
+ * Copies data from in to out until either we are out of data (returns null) or we hit one of the byte patterns identified by the <code>stoppers</code> parameter (returns the byte pattern
+ * matched). The byte pattern matched will NOT be copied to the output and will be un-read from the input.
+ *
+ * @param in the source to read bytes from
+ * @param out the destination to write bytes to
+ * @param maxBytes the maximum number of bytes to copy
+ * @param stoppers byte patterns which will cause the copy to stop if found
+ * @return the byte array matched, or null if end of stream was reached
+ * @throws IOException for issues reading or writing to underlying streams
+ */
+ public static byte[] copyExclusive(final InputStream in, final OutputStream out, final int maxBytes, final byte[]... stoppers) throws IOException {
+ if (stoppers.length == 0) {
+ return null;
+ }
+
+ int longest = 0;
+ NonThreadSafeCircularBuffer longestBuffer = null;
+ final List<NonThreadSafeCircularBuffer> circularBuffers = new ArrayList<>();
+ for (final byte[] stopper : stoppers) {
+ final NonThreadSafeCircularBuffer circularBuffer = new NonThreadSafeCircularBuffer(stopper);
+ if (stopper.length > longest) {
+ longest = stopper.length;
+ longestBuffer = circularBuffer;
+ circularBuffers.add(0, circularBuffer);
+ } else {
+ circularBuffers.add(circularBuffer);
+ }
+ }
+
+ long bytesRead = 0;
+ while (true) {
+ final int next = in.read();
+ if (next == -1) {
+ return null;
+ } else if (maxBytes > 0 && bytesRead++ > maxBytes) {
+ throw new BytePatternNotFoundException("Did not encounter any byte pattern that was expected; data does not appear to be in the expected format");
+ }
+
+ for (final NonThreadSafeCircularBuffer circ : circularBuffers) {
+ if (circ.addAndCompare((byte) next)) {
+ // The longest buffer has some data that may not have been written out yet; we need to make sure
+ // that we copy out those bytes.
+ final int bytesToCopy = longest - circ.getByteArray().length;
+ for (int i = 0; i < bytesToCopy; i++) {
+ final int oldestByte = longestBuffer.getOldestByte();
+ if (oldestByte != -1) {
+ out.write(oldestByte);
+ longestBuffer.addAndCompare((byte) 0);
+ }
+ }
+
+ return circ.getByteArray();
+ }
+ }
+
+ if (longestBuffer.isFilled()) {
+ out.write(longestBuffer.getOldestByte());
+ }
+ }
+ }
+
+ /**
+ * Skips the specified number of bytes from the InputStream
+ *
+ * If unable to skip that number of bytes, throws EOFException
+ *
+ * @param stream the stream to skip over
+ * @param bytesToSkip the number of bytes to skip
+ * @throws IOException if any issues reading or skipping underlying stream
+ */
+ public static void skip(final InputStream stream, final long bytesToSkip) throws IOException {
+ if (bytesToSkip <= 0) {
+ return;
+ }
+ long totalSkipped = 0L;
+
+ // If we have a FileInputStream, calling skip(1000000) will return 1000000 even if the file is only
+ // 3 bytes. As a result, we will skip 1 less than the number requested, and then read the last
+ // byte in order to make sure that we've consumed the number of bytes requested. We then check that
+ // the final byte, which we read, is not -1.
+ final long actualBytesToSkip = bytesToSkip - 1;
+ while (totalSkipped < actualBytesToSkip) {
+ final long skippedThisIteration = stream.skip(actualBytesToSkip - totalSkipped);
+ if (skippedThisIteration == 0) {
+ final int nextByte = stream.read();
+ if (nextByte == -1) {
+ throw new EOFException();
+ } else {
+ totalSkipped++;
+ }
+ }
+
+ totalSkipped += skippedThisIteration;
+ }
+
+ final int lastByte = stream.read();
+ if (lastByte == -1) {
+ throw new EOFException();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ZipOutputStream.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ZipOutputStream.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ZipOutputStream.java
new file mode 100644
index 0000000..d30af76
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ZipOutputStream.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io;
+
+import java.io.OutputStream;
+
+/**
+ * This class extends the {@link java.util.zip.ZipOutputStream} by providing a constructor that allows the user to specify the compression level. The default compression level is 1, as opposed to
+ * Java's default of 5.
+ */
+public class ZipOutputStream extends java.util.zip.ZipOutputStream {
+
+ public static final int DEFAULT_COMPRESSION_LEVEL = 1;
+
+ public ZipOutputStream(final OutputStream out) {
+ this(out, DEFAULT_COMPRESSION_LEVEL);
+ }
+
+ public ZipOutputStream(final OutputStream out, final int compressionLevel) {
+ super(out);
+ def.setLevel(compressionLevel);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/exception/BytePatternNotFoundException.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/exception/BytePatternNotFoundException.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/exception/BytePatternNotFoundException.java
new file mode 100644
index 0000000..5d08616
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/exception/BytePatternNotFoundException.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io.exception;
+
+import java.io.IOException;
+
+public class BytePatternNotFoundException extends IOException {
+
+ private static final long serialVersionUID = -4128911284318513973L;
+
+ public BytePatternNotFoundException(final String explanation) {
+ super(explanation);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/util/NonThreadSafeCircularBuffer.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/util/NonThreadSafeCircularBuffer.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/util/NonThreadSafeCircularBuffer.java
new file mode 100644
index 0000000..f18d824
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/util/NonThreadSafeCircularBuffer.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io.util;
+
+import java.util.Arrays;
+
+public class NonThreadSafeCircularBuffer {
+
+ private final byte[] lookingFor;
+ private final int[] buffer;
+ private int insertionPointer = 0;
+ private int bufferSize = 0;
+
+ public NonThreadSafeCircularBuffer(final byte[] lookingFor) {
+ this.lookingFor = lookingFor;
+ buffer = new int[lookingFor.length];
+ Arrays.fill(buffer, -1);
+ }
+
+ public byte[] getByteArray() {
+ return lookingFor;
+ }
+
+ /**
+ * Returns the oldest byte in the buffer
+ *
+ * @return the oldest byte
+ */
+ public int getOldestByte() {
+ return buffer[insertionPointer];
+ }
+
+ public boolean isFilled() {
+ return bufferSize >= buffer.length;
+ }
+
+ public boolean addAndCompare(final byte data) {
+ buffer[insertionPointer] = data;
+ insertionPointer = (insertionPointer + 1) % lookingFor.length;
+
+ bufferSize++;
+ if (bufferSize < lookingFor.length) {
+ return false;
+ }
+
+ for (int i = 0; i < lookingFor.length; i++) {
+ final byte compare = (byte) buffer[(insertionPointer + i) % lookingFor.length];
+ if (compare != lookingFor[i]) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/BooleanHolder.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/BooleanHolder.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/BooleanHolder.java
new file mode 100644
index 0000000..92061e0
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/BooleanHolder.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+public class BooleanHolder extends ObjectHolder<Boolean> {
+
+ public BooleanHolder(final boolean initialValue) {
+ super(initialValue);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/EscapeUtils.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/EscapeUtils.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/EscapeUtils.java
new file mode 100644
index 0000000..46739e3
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/EscapeUtils.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+public class EscapeUtils {
+
+ /**
+ * Escapes the specified html by replacing &, <, >, ", ', / with their corresponding html entity. If html is null, null is returned.
+ *
+ * @param html to escape
+ * @return escaped html
+ */
+ public static String escapeHtml(String html) {
+ if (html == null) {
+ return null;
+ }
+
+ html = html.replace("&", "&");
+ html = html.replace("<", "<");
+ html = html.replace(">", ">");
+ html = html.replace("\"", """);
+ html = html.replace("'", "'");
+ html = html.replace("/", "/");
+
+ return html;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/FormatUtils.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/FormatUtils.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/FormatUtils.java
new file mode 100644
index 0000000..03afec0
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/FormatUtils.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import java.text.NumberFormat;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Locale;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class FormatUtils {
+
+ private static final String UNION = "|";
+
+ // for Data Sizes
+ private static final double BYTES_IN_KILOBYTE = 1024;
+ private static final double BYTES_IN_MEGABYTE = BYTES_IN_KILOBYTE * 1024;
+ private static final double BYTES_IN_GIGABYTE = BYTES_IN_MEGABYTE * 1024;
+ private static final double BYTES_IN_TERABYTE = BYTES_IN_GIGABYTE * 1024;
+
+ // for Time Durations
+ private static final String NANOS = join(UNION, "ns", "nano", "nanos", "nanoseconds");
+ private static final String MILLIS = join(UNION, "ms", "milli", "millis", "milliseconds");
+ private static final String SECS = join(UNION, "s", "sec", "secs", "second", "seconds");
+ private static final String MINS = join(UNION, "m", "min", "mins", "minute", "minutes");
+ private static final String HOURS = join(UNION, "h", "hr", "hrs", "hour", "hours");
+ private static final String DAYS = join(UNION, "d", "day", "days");
+
+ private static final String VALID_TIME_UNITS = join(UNION, NANOS, MILLIS, SECS, MINS, HOURS, DAYS);
+ public static final String TIME_DURATION_REGEX = "(\\d+)\\s*(" + VALID_TIME_UNITS + ")";
+ public static final Pattern TIME_DURATION_PATTERN = Pattern.compile(TIME_DURATION_REGEX);
+
+ /**
+ * Formats the specified count by adding commas.
+ *
+ * @param count the value to add commas to
+ * @return the string representation of the given value with commas included
+ */
+ public static String formatCount(final long count) {
+ return NumberFormat.getIntegerInstance().format(count);
+ }
+
+ /**
+ * Formats the specified duration in 'mm:ss.SSS' format.
+ *
+ * @param sourceDuration the duration to format
+ * @param sourceUnit the unit to interpret the duration
+ * @return representation of the given time data in minutes/seconds
+ */
+ public static String formatMinutesSeconds(final long sourceDuration, final TimeUnit sourceUnit) {
+ final long millis = TimeUnit.MILLISECONDS.convert(sourceDuration, sourceUnit);
+ final SimpleDateFormat formatter = new SimpleDateFormat("mm:ss.SSS", Locale.US);
+ return formatter.format(new Date(millis));
+ }
+
+ /**
+ * Formats the specified duration in 'HH:mm:ss.SSS' format.
+ *
+ * @param sourceDuration the duration to format
+ * @param sourceUnit the unit to interpret the duration
+ * @return representation of the given time data in hours/minutes/seconds
+ */
+ public static String formatHoursMinutesSeconds(final long sourceDuration, final TimeUnit sourceUnit) {
+ final long millis = TimeUnit.MILLISECONDS.convert(sourceDuration, sourceUnit);
+ final long millisInHour = TimeUnit.MILLISECONDS.convert(1, TimeUnit.HOURS);
+ final int hours = (int) (millis / millisInHour);
+ final long whatsLeft = millis - hours * millisInHour;
+
+ return pad(hours) + ":" + new SimpleDateFormat("mm:ss.SSS", Locale.US).format(new Date(whatsLeft));
+ }
+
+ private static String pad(final int val) {
+ return (val < 10) ? "0" + val : String.valueOf(val);
+ }
+
+ /**
+ * Formats the specified data size in human readable format.
+ *
+ * @param dataSize Data size in bytes
+ * @return Human readable format
+ */
+ public static String formatDataSize(final double dataSize) {
+ // initialize the formatter
+ final NumberFormat format = NumberFormat.getNumberInstance();
+ format.setMaximumFractionDigits(2);
+
+ // check terabytes
+ double dataSizeToFormat = dataSize / BYTES_IN_TERABYTE;
+ if (dataSizeToFormat > 1) {
+ return format.format(dataSizeToFormat) + " TB";
+ }
+
+ // check gigabytes
+ dataSizeToFormat = dataSize / BYTES_IN_GIGABYTE;
+ if (dataSizeToFormat > 1) {
+ return format.format(dataSizeToFormat) + " GB";
+ }
+
+ // check megabytes
+ dataSizeToFormat = dataSize / BYTES_IN_MEGABYTE;
+ if (dataSizeToFormat > 1) {
+ return format.format(dataSizeToFormat) + " MB";
+ }
+
+ // check kilobytes
+ dataSizeToFormat = dataSize / BYTES_IN_KILOBYTE;
+ if (dataSizeToFormat > 1) {
+ return format.format(dataSizeToFormat) + " KB";
+ }
+
+ // default to bytes
+ return format.format(dataSize) + " bytes";
+ }
+
+ public static long getTimeDuration(final String value, final TimeUnit desiredUnit) {
+ final Matcher matcher = TIME_DURATION_PATTERN.matcher(value.toLowerCase());
+ if (!matcher.matches()) {
+ throw new IllegalArgumentException("Value '" + value + "' is not a valid Time Duration");
+ }
+
+ final String duration = matcher.group(1);
+ final String units = matcher.group(2);
+ TimeUnit specifiedTimeUnit = null;
+ switch (units.toLowerCase()) {
+ case "ns":
+ case "nano":
+ case "nanos":
+ case "nanoseconds":
+ specifiedTimeUnit = TimeUnit.NANOSECONDS;
+ break;
+ case "ms":
+ case "milli":
+ case "millis":
+ case "milliseconds":
+ specifiedTimeUnit = TimeUnit.MILLISECONDS;
+ break;
+ case "s":
+ case "sec":
+ case "secs":
+ case "second":
+ case "seconds":
+ specifiedTimeUnit = TimeUnit.SECONDS;
+ break;
+ case "m":
+ case "min":
+ case "mins":
+ case "minute":
+ case "minutes":
+ specifiedTimeUnit = TimeUnit.MINUTES;
+ break;
+ case "h":
+ case "hr":
+ case "hrs":
+ case "hour":
+ case "hours":
+ specifiedTimeUnit = TimeUnit.HOURS;
+ break;
+ case "d":
+ case "day":
+ case "days":
+ specifiedTimeUnit = TimeUnit.DAYS;
+ break;
+ }
+
+ final long durationVal = Long.parseLong(duration);
+ return desiredUnit.convert(durationVal, specifiedTimeUnit);
+ }
+
+ public static String formatUtilization(final double utilization) {
+ return utilization + "%";
+ }
+
+ private static String join(final String delimiter, final String... values) {
+ if (values.length == 0) {
+ return "";
+ } else if (values.length == 1) {
+ return values[0];
+ }
+
+ final StringBuilder sb = new StringBuilder();
+ sb.append(values[0]);
+ for (int i = 1; i < values.length; i++) {
+ sb.append(delimiter).append(values[i]);
+ }
+
+ return sb.toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/IntegerHolder.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/IntegerHolder.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/IntegerHolder.java
new file mode 100644
index 0000000..213bbc0
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/IntegerHolder.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+public class IntegerHolder extends ObjectHolder<Integer> {
+
+ public IntegerHolder(final int initialValue) {
+ super(initialValue);
+ }
+
+ public int addAndGet(final int delta) {
+ final int curValue = get();
+ final int newValue = curValue + delta;
+ set(newValue);
+ return newValue;
+ }
+
+ public int getAndAdd(final int delta) {
+ final int curValue = get();
+ final int newValue = curValue + delta;
+ set(newValue);
+ return curValue;
+ }
+
+ public int incrementAndGet() {
+ return addAndGet(1);
+ }
+
+ public int getAndIncrement() {
+ return getAndAdd(1);
+ }
+
+ public int decrementAndGet() {
+ return addAndGet(-1);
+ }
+
+ public int getAndDecrement() {
+ return getAndAdd(-1);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/LongHolder.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/LongHolder.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/LongHolder.java
new file mode 100644
index 0000000..fa2d063
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/LongHolder.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+/**
+ * Wraps a Long value so that it can be declared <code>final</code> and still be accessed from which inner classes; the functionality is similar to that of an AtomicLong, but operations on this class
+ * are not atomic. This results in greater performance when the atomicity is not needed.
+ */
+public class LongHolder extends ObjectHolder<Long> {
+
+ public LongHolder(final long initialValue) {
+ super(initialValue);
+ }
+
+ public long addAndGet(final long delta) {
+ final long curValue = get();
+ final long newValue = curValue + delta;
+ set(newValue);
+ return newValue;
+ }
+
+ public long getAndAdd(final long delta) {
+ final long curValue = get();
+ final long newValue = curValue + delta;
+ set(newValue);
+ return curValue;
+ }
+
+ public long incrementAndGet() {
+ return addAndGet(1);
+ }
+
+ public long getAndIncrement() {
+ return getAndAdd(1);
+ }
+
+ public long decrementAndGet() {
+ return addAndGet(-1L);
+ }
+
+ public long getAndDecrement() {
+ return getAndAdd(-1L);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/NaiveSearchRingBuffer.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/NaiveSearchRingBuffer.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/NaiveSearchRingBuffer.java
new file mode 100644
index 0000000..0c6c575
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/NaiveSearchRingBuffer.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import java.util.Arrays;
+
+/**
+ * <p>
+ * A RingBuffer that can be used to scan byte sequences for subsequences.
+ * </p>
+ *
+ * <p>
+ * This class implements an efficient naive search algorithm, which allows the user of the library to identify byte sequences in a stream on-the-fly so that the stream can be segmented without having
+ * to buffer the data.
+ * </p>
+ *
+ * <p>
+ * The intended usage paradigm is:
+ * <code>
+ * <pre>
+ * final byte[] searchSequence = ...;
+ * final CircularBuffer buffer = new CircularBuffer(searchSequence);
+ * while ((int nextByte = in.read()) > 0) {
+ * if ( buffer.addAndCompare(nextByte) ) {
+ * // This byte is the last byte in the given sequence
+ * } else {
+ * // This byte does not complete the given sequence
+ * }
+ * }
+ * </pre>
+ * </code>
+ * </p>
+ */
+public class NaiveSearchRingBuffer {
+
+ private final byte[] lookingFor;
+ private final int[] buffer;
+ private int insertionPointer = 0;
+ private int bufferSize = 0;
+
+ public NaiveSearchRingBuffer(final byte[] lookingFor) {
+ this.lookingFor = lookingFor;
+ this.buffer = new int[lookingFor.length];
+ Arrays.fill(buffer, -1);
+ }
+
+ /**
+ * @return the contents of the internal buffer, which represents the last X bytes added to the buffer, where X is the minimum of the number of bytes added to the buffer or the length of the byte
+ * sequence for which we are looking
+ */
+ public byte[] getBufferContents() {
+ final int contentLength = Math.min(lookingFor.length, bufferSize);
+ final byte[] contents = new byte[contentLength];
+ for (int i = 0; i < contentLength; i++) {
+ final byte nextByte = (byte) buffer[(insertionPointer + i) % lookingFor.length];
+ contents[i] = nextByte;
+ }
+ return contents;
+ }
+
+ /**
+ * @return the oldest byte in the buffer
+ */
+ public int getOldestByte() {
+ return buffer[insertionPointer];
+ }
+
+ /**
+ * @return <code>true</code> if the number of bytes that have been added to the buffer is at least equal to the length of the byte sequence for which we are searching
+ */
+ public boolean isFilled() {
+ return bufferSize >= buffer.length;
+ }
+
+ /**
+ * Clears the internal buffer so that a new search may begin
+ */
+ public void clear() {
+ Arrays.fill(buffer, -1);
+ insertionPointer = 0;
+ bufferSize = 0;
+ }
+
+ /**
+ * Add the given byte to the buffer and notify whether or not the byte completes the desired byte sequence.
+ *
+ * @param data the data to add to the buffer
+ * @return <code>true</code> if this byte completes the byte sequence, <code>false</code> otherwise.
+ */
+ public boolean addAndCompare(final byte data) {
+ buffer[insertionPointer] = data;
+ insertionPointer = (insertionPointer + 1) % lookingFor.length;
+
+ bufferSize++;
+ if (bufferSize < lookingFor.length) {
+ return false;
+ }
+
+ for (int i = 0; i < lookingFor.length; i++) {
+ final byte compare = (byte) buffer[(insertionPointer + i) % lookingFor.length];
+ if (compare != lookingFor[i]) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/ObjectHolder.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/ObjectHolder.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/ObjectHolder.java
new file mode 100644
index 0000000..12a887c
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/ObjectHolder.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+/**
+ * A bean that holds a single value of type T.
+ *
+ */
+public class ObjectHolder<T> {
+
+ private T value;
+
+ public ObjectHolder(final T initialValue) {
+ this.value = initialValue;
+ }
+
+ public T get() {
+ return value;
+ }
+
+ public void set(T value) {
+ this.value = value;
+ }
+}
[34/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java b/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java
new file mode 100644
index 0000000..131bcde
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java
@@ -0,0 +1,1176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.Query.Range;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.exception.AttributeExpressionLanguageException;
+import org.apache.nifi.attribute.expression.language.exception.AttributeExpressionLanguageParsingException;
+import org.apache.nifi.expression.AttributeExpression.ResultType;
+import org.apache.nifi.flowfile.FlowFile;
+import org.antlr.runtime.tree.Tree;
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+public class TestQuery {
+
+ @Test
+ public void testCompilation() {
+ assertInvalid("${attr:uuid()}");
+ assertInvalid("${attr:indexOf(length())}");
+ assertValid("${UUID()}");
+ assertInvalid("${UUID():nextInt()}");
+ assertValid("${nextInt()}");
+ assertValid("${now():format('yyyy/MM/dd')}");
+ assertInvalid("${attr:times(3)}");
+ assertValid("${attr:toNumber():multiply(3)}");
+ assertValid("${hostname()}");
+ assertValid("${literal(3)}");
+ // left here because it's convenient for looking at the output
+ //System.out.println(Query.compile("").evaluate(null));
+ }
+
+ private void assertValid(final String query) {
+ try {
+ Query.compile(query);
+ } catch (final Exception e) {
+ e.printStackTrace();
+ Assert.fail("Expected query to be valid, but it failed to compile due to " + e);
+ }
+ }
+
+ private void assertInvalid(final String query) {
+ try {
+ Query.compile(query);
+ Assert.fail("Expected query to be invalid, but it did compile");
+ } catch (final Exception e) {
+ }
+ }
+
+ @Test
+ public void testIsValidExpression() {
+ Query.validateExpression("${abc:substring(${xyz:length()})}", false);
+ Query.isValidExpression("${now():format('yyyy-MM-dd')}");
+
+ try {
+ Query.validateExpression("$${attr}", false);
+ Assert.fail("invalid query validated");
+ } catch (final AttributeExpressionLanguageParsingException e) {
+ }
+
+ Query.validateExpression("$${attr}", true);
+
+ Query.validateExpression("${filename:startsWith('T8MTXBC')\n"
+ + ":or( ${filename:startsWith('C4QXABC')} )\n"
+ + ":or( ${filename:startsWith('U6CXEBC')} )"
+ + ":or( ${filename:startsWith('KYM3ABC')} )}", false);
+ }
+
+ @Test
+ public void testCompileEmbedded() {
+ final String expression = "${x:equals( ${y} )}";
+ final Query query = Query.compile(expression);
+ final Tree tree = query.getTree();
+ System.out.println(printTree(tree));
+
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("x", "x");
+ attributes.put("y", "x");
+ final String result = Query.evaluateExpressions(expression, attributes, null);
+ assertEquals("true", result);
+
+ Query.validateExpression(expression, false);
+ }
+
+ private String printTree(final Tree tree) {
+ final StringBuilder sb = new StringBuilder();
+ printTree(tree, 0, sb);
+
+ return sb.toString();
+ }
+
+ private void printTree(final Tree tree, final int spaces, final StringBuilder sb) {
+ for (int i = 0; i < spaces; i++) {
+ sb.append(" ");
+ }
+
+ if (tree.getText().trim().isEmpty()) {
+ sb.append(tree.toString()).append("\n");
+ } else {
+ sb.append(tree.getText()).append("\n");
+ }
+
+ for (int i = 0; i < tree.getChildCount(); i++) {
+ printTree(tree.getChild(i), spaces + 2, sb);
+ }
+ }
+
+ @Test
+ public void testEscape() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("attr", "My Value");
+ attributes.put("${xx}", "hello");
+
+ assertEquals("My Value", evaluateQueryForEscape("${attr}", attributes));
+ assertEquals("${attr}", evaluateQueryForEscape("$${attr}", attributes));
+ assertEquals("$My Value", evaluateQueryForEscape("$$${attr}", attributes));
+ assertEquals("$${attr}", evaluateQueryForEscape("$$$${attr}", attributes));
+ assertEquals("$$My Value", evaluateQueryForEscape("$$$$${attr}", attributes));
+ }
+
+ @Test
+ public void testWithBackSlashes() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("x", "C:\\test\\1.txt");
+ attributes.put("y", "y\ny");
+
+ final String query = "${x:substringAfterLast( '/' ):substringAfterLast( '\\\\' )}";
+ verifyEquals(query, attributes, "1.txt");
+ attributes.put("x", "C:/test/1.txt");
+ verifyEquals(query, attributes, "1.txt");
+
+ verifyEquals("${y:equals('y\\ny')}", attributes, Boolean.TRUE);
+ }
+
+ @Test
+ public void testWithTicksOutside() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("attr", "My Value");
+
+ assertEquals(1, Query.extractExpressionRanges("\"${attr}").size());
+ assertEquals(1, Query.extractExpressionRanges("'${attr}").size());
+ assertEquals(1, Query.extractExpressionRanges("'${attr}'").size());
+ assertEquals(1, Query.extractExpressionRanges("${attr}").size());
+
+ assertEquals("'My Value'", Query.evaluateExpressions("'${attr}'", attributes, null));
+ assertEquals("'My Value", Query.evaluateExpressions("'${attr}", attributes, null));
+ }
+
+ @Test
+ @Ignore("Depends on TimeZone")
+ public void testDateToNumber() {
+ final Query query = Query.compile("${dateTime:toDate('yyyy/MM/dd HH:mm:ss.SSS'):toNumber()}");
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("dateTime", "2013/11/18 10:22:27.678");
+
+ final QueryResult<?> result = query.evaluate(attributes);
+ assertEquals(ResultType.NUMBER, result.getResultType());
+ assertEquals(1384788147678L, result.getValue());
+ }
+
+ @Test
+ public void testAddOneDayToDate() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("dateTime", "2013/11/18 10:22:27.678");
+
+ verifyEquals("${dateTime:toDate('yyyy/MM/dd HH:mm:ss.SSS'):toNumber():plus(86400000):toDate():format('yyyy/MM/dd HH:mm:ss.SSS')}", attributes, "2013/11/19 10:22:27.678");
+ verifyEquals("${dateTime:toDate('yyyy/MM/dd HH:mm:ss.SSS'):plus(86400000):format('yyyy/MM/dd HH:mm:ss.SSS')}", attributes, "2013/11/19 10:22:27.678");
+ }
+
+ @Test
+ @Ignore("Requires specific locale")
+ public void implicitDateConversion() {
+ final Date date = new Date();
+ final Query query = Query.compile("${dateTime:format('yyyy/MM/dd HH:mm:ss.SSS')}");
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("dateTime", date.toString());
+
+ // the date.toString() above will end up truncating the milliseconds. So remove millis from the Date before
+ // formatting it
+ final SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss.SSS", Locale.US);
+ final long millis = date.getTime() % 1000L;
+ final Date roundedToNearestSecond = new Date(date.getTime() - millis);
+ final String formatted = sdf.format(roundedToNearestSecond);
+
+ final QueryResult<?> result = query.evaluate(attributes);
+ assertEquals(ResultType.STRING, result.getResultType());
+ assertEquals(formatted, result.getValue());
+ }
+
+ @Test
+ public void testEmbeddedExpressionsAndQuotes() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("x", "abc");
+ attributes.put("a", "abc");
+
+ verifyEquals("${x:equals(${a})}", attributes, true);
+
+ Query.validateExpression("${x:equals('${a}')}", false);
+ assertEquals("true", Query.evaluateExpressions("${x:equals('${a}')}", attributes, null));
+
+ Query.validateExpression("${x:equals(\"${a}\")}", false);
+ assertEquals("true", Query.evaluateExpressions("${x:equals(\"${a}\")}", attributes, null));
+ }
+
+ @Test
+ public void testJoin() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("a.a", "a");
+ attributes.put("a.b", "b");
+ attributes.put("a.c", "c");
+ verifyEquals("${allAttributes( 'a.a', 'a.b', 'a.c' ):join(', ')}", attributes, "a, b, c");
+ verifyEquals("${x:join(', ')}", attributes, "");
+ verifyEquals("${a.a:join(', ')}", attributes, "a");
+ verifyEquals("${allAttributes( 'x', 'y' ):join(',')}", attributes, ",");
+ }
+
+ @Test(expected = AttributeExpressionLanguageException.class)
+ public void testCannotCombineWithNonReducingFunction() {
+ Query.compileTree("${allAttributes( 'a.1' ):plus(1)}");
+ }
+
+ @Test
+ public void testIsEmpty() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("a", "a");
+ attributes.put("b", "");
+ attributes.put("c", " \n");
+
+ verifyEquals("${a:isEmpty()}", attributes, false);
+ verifyEquals("${b:isEmpty()}", attributes, true);
+ verifyEquals("${c:isEmpty()}", attributes, true);
+ verifyEquals("${d:isEmpty()}", attributes, true);
+ }
+
+ @Test
+ public void testReplaceEmpty() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("a", "a");
+ attributes.put("b", "");
+ attributes.put("c", " \n");
+
+ verifyEquals("${a:replaceEmpty('c')}", attributes, "a");
+ verifyEquals("${b:replaceEmpty('c')}", attributes, "c");
+ verifyEquals("${c:replaceEmpty('c')}", attributes, "c");
+ verifyEquals("${d:replaceEmpty('c')}", attributes, "c");
+ }
+
+ @Test
+ public void testCount() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("a", "a");
+ attributes.put("b", "abc");
+ attributes.put("c", " \n");
+ attributes.put("n1", "111");
+ attributes.put("n2", "222");
+ attributes.put("n3", "333333");
+
+ verifyEquals("${allMatchingAttributes( '.*' ):count()}", attributes, 6L);
+ verifyEquals("${allMatchingAttributes( '.*' ):length():gt(2):count()}", attributes, 5L);
+ verifyEquals("${allMatchingAttributes( 'n.*' ):plus(1):count()}", attributes, 3L);
+ }
+
+ @Test
+ public void testCurlyBracesInQuotes() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("attr", "My Valuee");
+
+ assertEquals("Val", evaluateQueryForEscape("${attr:replaceAll('My (Val)ue{1,2}', '$1')}", attributes));
+ assertEquals("Val", evaluateQueryForEscape("${attr:replaceAll(\"My (Val)ue{1,2}\", '$1')}", attributes));
+ }
+
+ private String evaluateQueryForEscape(final String queryString, final Map<String, String> attributes) {
+ final FlowFile mockFlowFile = Mockito.mock(FlowFile.class);
+ Mockito.when(mockFlowFile.getAttributes()).thenReturn(attributes);
+ Mockito.when(mockFlowFile.getId()).thenReturn(1L);
+ Mockito.when(mockFlowFile.getEntryDate()).thenReturn(System.currentTimeMillis());
+ Mockito.when(mockFlowFile.getSize()).thenReturn(1L);
+ Mockito.when(mockFlowFile.getLineageIdentifiers()).thenReturn(new HashSet<String>());
+ Mockito.when(mockFlowFile.getLineageStartDate()).thenReturn(System.currentTimeMillis());
+ return Query.evaluateExpressions(queryString, mockFlowFile);
+ }
+
+ @Test
+ public void testGetAttributeValue() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("attr", "My Value");
+ verifyEquals("${attr}", attributes, "My Value");
+ }
+
+ @Test
+ public void testGetAttributeValueEmbedded() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("attr", "XX ");
+ attributes.put("XX", "My Value");
+ verifyEquals("${${attr:trim()}}", attributes, "My Value");
+ }
+
+ @Test
+ public void testSimpleSubstring() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("attr", "My Value");
+ verifyEquals("${attr:substring(2, 5)}", attributes, " Va");
+ }
+
+ @Test
+ public void testCallToFunctionWithSubjectResultOfAnotherFunctionCall() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("attr", " My Value ");
+ verifyEquals("${attr:trim():substring(2, 5)}", attributes, " Va");
+ }
+
+ @Test
+ public void testProblematic1() {
+ // There was a bug that prevented this expression from compiling. This test just verifies that it now compiles.
+ final String queryString = "${xx:append( \"120101\" ):toDate( 'yyMMddHHmmss' ):format( \"yy-MM-dd’T’HH:mm:ss\") }";
+ Query.compile(queryString);
+ }
+
+ @Test
+ public void testEquals() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("attr", " XX ");
+ verifyEquals("${attr:trim():equals('XX')}", attributes, true);
+ }
+
+ @Test
+ public void testDeeplyEmbedded() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("x", "false");
+ attributes.put("abc", "a");
+ attributes.put("a", "a");
+
+ verifyEquals("${x:or( ${${abc}:length():equals(1)} )}", attributes, true);
+ }
+
+ @Test
+ public void testExtractExpressionRanges() {
+ assertEquals(29, Query.extractExpressionRanges("${hello:equals( $${goodbye} )}").get(0).getEnd());
+
+ List<Range> ranges = Query.extractExpressionRanges("hello");
+ assertTrue(ranges.isEmpty());
+
+ ranges = Query.extractExpressionRanges("${hello");
+ assertTrue(ranges.isEmpty());
+
+ ranges = Query.extractExpressionRanges("hello}");
+ assertTrue(ranges.isEmpty());
+
+ ranges = Query.extractExpressionRanges("$${hello");
+ assertTrue(ranges.isEmpty());
+
+ ranges = Query.extractExpressionRanges("$he{ll}o");
+ assertTrue(ranges.isEmpty());
+
+ ranges = Query.extractExpressionRanges("${hello}");
+ assertEquals(1, ranges.size());
+ Range range = ranges.get(0);
+ assertEquals(0, range.getStart());
+ assertEquals(7, range.getEnd());
+
+ ranges = Query.extractExpressionRanges("${hello:equals( ${goodbye} )}");
+ assertEquals(1, ranges.size());
+ range = ranges.get(0);
+ assertEquals(0, range.getStart());
+ assertEquals(28, range.getEnd());
+
+ ranges = Query.extractExpressionRanges("${hello:equals( $${goodbye} )}");
+ assertEquals(1, ranges.size());
+ range = ranges.get(0);
+ assertEquals(0, range.getStart());
+ assertEquals(29, range.getEnd());
+
+ ranges = Query.extractExpressionRanges("${hello:equals( $${goodbye} )} or just hi, ${bob:or(${jerry})}");
+ assertEquals(2, ranges.size());
+ range = ranges.get(0);
+ assertEquals(0, range.getStart());
+ assertEquals(29, range.getEnd());
+
+ range = ranges.get(1);
+ assertEquals(43, range.getStart());
+ assertEquals(61, range.getEnd());
+
+ ranges = Query.extractExpressionRanges("${hello:equals( ${goodbye} )} or just hi, ${bob}, are you ${bob.age:toNumber()} yet? $$$${bob}");
+ assertEquals(3, ranges.size());
+ range = ranges.get(0);
+ assertEquals(0, range.getStart());
+ assertEquals(28, range.getEnd());
+
+ range = ranges.get(1);
+ assertEquals(42, range.getStart());
+ assertEquals(47, range.getEnd());
+
+ range = ranges.get(2);
+ assertEquals(58, range.getStart());
+ assertEquals(78, range.getEnd());
+
+ ranges = Query.extractExpressionRanges("${x:matches( '.{4}' )}");
+ assertEquals(1, ranges.size());
+ range = ranges.get(0);
+ assertEquals(0, range.getStart());
+ assertEquals(21, range.getEnd());
+ }
+
+ @Test
+ public void testExtractExpressionTypes() {
+ final List<ResultType> types = Query.extractResultTypes("${hello:equals( ${goodbye} )} or just hi, ${bob}, are you ${bob.age:toNumber()} yet? $$$${bob}");
+ assertEquals(3, types.size());
+ assertEquals(ResultType.BOOLEAN, types.get(0));
+ assertEquals(ResultType.STRING, types.get(1));
+ assertEquals(ResultType.NUMBER, types.get(2));
+ }
+
+ @Test
+ public void testEqualsEmbedded() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("x", "hello");
+ attributes.put("y", "good-bye");
+
+ verifyEquals("${x:equals( ${y} )}", attributes, false);
+
+ attributes.put("y", "hello");
+ verifyEquals("${x:equals( ${y} )}", attributes, true);
+
+ attributes.put("x", "4");
+ attributes.put("y", "3");
+ attributes.put("z", "1");
+ attributes.put("h", "100");
+ verifyEquals("${x:toNumber():lt( ${y:toNumber():plus( ${h:toNumber()} )} )}", attributes, true);
+ verifyEquals("${h:toNumber():ge( ${y:toNumber():plus( ${z:toNumber()} )} )}", attributes, true);
+ verifyEquals("${x:toNumber():equals( ${y:toNumber():plus( ${z:toNumber()} )} )}", attributes, true);
+
+ attributes.put("x", "88");
+ verifyEquals("${x:toNumber():gt( ${y:toNumber():plus( ${z:toNumber()} )} )}", attributes, true);
+
+ attributes.put("y", "88");
+ assertEquals("true", Query.evaluateExpressions("${x:equals( '${y}' )}", attributes, null));
+ }
+
+ @Test
+ public void testComplicatedEmbeddedExpressions() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("fox", "quick, brown");
+ attributes.put("dog", "lazy");
+
+ verifyEquals("${fox:substring( ${ 'dog' :substring(2):length()}, 5 ):equals( 'ick' )}", attributes, true);
+ verifyEquals("${fox:substring( ${ 'dog' :substring(2):length()}, 5 ):equals( 'ick' )}", attributes, true);
+ }
+
+ @Test
+ public void testQuotingQuotes() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("xx", "say 'hi'");
+
+ String query = "${xx:replaceAll( \"'.*'\", '\\\"hello\\\"' )}";
+ verifyEquals(query, attributes, "say \"hello\"");
+
+ query = "${xx:replace( \"'\", '\"')}";
+ verifyEquals(query, attributes, "say \"hi\"");
+
+ query = "${xx:replace( '\\'', '\"')}";
+ System.out.println(query);
+ verifyEquals(query, attributes, "say \"hi\"");
+ }
+
+ @Test
+ public void testDoubleQuotesWithinSingleQuotes() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("xx", "say 'hi'");
+
+ final String query = "${xx:replace( \"'hi'\", '\\\"hello\\\"' )}";
+ System.out.println(query);
+ verifyEquals(query, attributes, "say \"hello\"");
+ }
+
+ @Test
+ public void testEscapeQuotes() {
+ final long timestamp = 1403620278642L;
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("date", String.valueOf(timestamp));
+
+ final String format = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'";
+
+ final String query = "startDateTime=\"${date:toNumber():toDate():format(\"" + format + "\")}\"";
+ final String result = Query.evaluateExpressions(query, attributes, null);
+
+ final String expectedTime = new SimpleDateFormat(format, Locale.US).format(timestamp);
+ assertEquals("startDateTime=\"" + expectedTime + "\"", result);
+
+ final List<Range> ranges = Query.extractExpressionRanges(query);
+ assertEquals(1, ranges.size());
+ }
+
+ @Test
+ public void testDateConversion() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("date", "1403620278642");
+
+ verifyEquals("${date:format('yyyy')}", attributes, "2014");
+ verifyEquals("${date:toDate():format('yyyy')}", attributes, "2014");
+ verifyEquals("${date:toNumber():format('yyyy')}", attributes, "2014");
+ verifyEquals("${date:toNumber():toDate():format('yyyy')}", attributes, "2014");
+ verifyEquals("${date:toDate():toNumber():format('yyyy')}", attributes, "2014");
+ verifyEquals("${date:toDate():toNumber():toDate():toNumber():toDate():toNumber():format('yyyy')}", attributes, "2014");
+ }
+
+ @Test
+ public void testSingleLetterAttribute() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("A", "0123456789");
+
+ verifyEquals("${A}", attributes, "0123456789");
+ verifyEquals("${'A'}", attributes, "0123456789");
+ }
+
+ @Test
+ public void testImplicitConversions() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("A", "0123456789");
+ attributes.put("b", "true");
+ attributes.put("c", "false");
+ attributes.put("d", "Quick Brown Fox");
+ attributes.put("F", "-48");
+ attributes.put("n", "2014/04/04 00:00:00");
+
+ final Calendar cal = Calendar.getInstance();
+ cal.set(Calendar.YEAR, 2014);
+ cal.set(Calendar.MONTH, 3);
+ cal.set(Calendar.DAY_OF_MONTH, 4);
+ cal.set(Calendar.HOUR, 0);
+ cal.set(Calendar.MINUTE, 0);
+ cal.set(Calendar.SECOND, 45);
+
+ final String dateString = cal.getTime().toString();
+ attributes.put("z", dateString);
+
+ verifyEquals("${A:plus(4)}", attributes, 123456793L);
+ verifyEquals("${A:plus( ${F} )}", attributes, 123456741L);
+
+ verifyEquals("${F:lt( ${A} )}", attributes, true);
+ verifyEquals("${A:substring(2,3):plus(21):substring(1,2):plus(0)}", attributes, 3L);
+ verifyEquals("${n:format( 'yyyy' )}", attributes, "2014");
+ verifyEquals("${z:format( 'yyyy' )}", attributes, "2014");
+
+ attributes.put("n", "2014/04/04 00:00:00.045");
+ verifyEquals("${n:format( 'yyyy' ):append(','):append( ${n:format( 'SSS' )} )}", attributes, "2014,045");
+ }
+
+ @Test
+ public void testNewLinesAndTabsInQuery() {
+ final String query = "${ abc:equals('abc'):or( \n\t${xx:isNull()}\n) }";
+ assertEquals(ResultType.BOOLEAN, Query.getResultType(query));
+ Query.validateExpression(query, false);
+ assertEquals("true", Query.evaluateExpressions(query));
+ }
+
+ @Test
+ public void testAttributeReferencesWithWhiteSpace() {
+ final Map<String, String> attrs = new HashMap<>();
+ attrs.put("a b c,d", "abc");
+
+ final String query = "${ 'a b c,d':equals('abc') }";
+ verifyEquals(query, attrs, true);
+ }
+
+ @Test
+ public void testComments() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("abc", "xyz");
+
+ final String expression
+ = "# hello, world\n"
+ + "${# ref attr\n"
+ + "\t"
+ + "abc"
+ + "\t"
+ + "#end ref attr\n"
+ + "}";
+
+ Query query = Query.compile(expression);
+ QueryResult<?> result = query.evaluate(attributes);
+ assertEquals(ResultType.STRING, result.getResultType());
+ assertEquals("xyz", result.getValue());
+
+ query = Query.compile("${abc:append('# hello') #good-bye \n}");
+ result = query.evaluate(attributes);
+ assertEquals(ResultType.STRING, result.getResultType());
+ assertEquals("xyz# hello", result.getValue());
+ }
+
+ @Test
+ public void testAppendPrepend() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("attr", "XX");
+ attributes.put("YXXX", "bingo");
+
+ verifyEquals("${${attr:append('X'):prepend('Y')}}", attributes, "bingo");
+ }
+
+ @Test
+ public void testIsNull() {
+ final Map<String, String> attributes = new HashMap<>();
+ verifyEquals("${attr:isNull()}", attributes, true);
+ }
+
+ @Test
+ public void testNotNull() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("attr", "");
+
+ verifyEquals("${attr:notNull()}", attributes, true);
+ }
+
+ @Test
+ public void testIsNullOrLengthEquals0() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("abc", "");
+ attributes.put("xyz", "xyz");
+ attributes.put("xx", " ");
+
+ verifyEquals("${abc:isNull():or( ${abc:length():equals(0)} )}", attributes, true);
+ verifyEquals("${xyz:isNull():or( ${xyz:length():equals(0)} )}", attributes, false);
+ verifyEquals("${none:isNull():or( ${none:length():equals(0)} )}", attributes, true);
+ verifyEquals("${xx:isNull():or( ${xx:trim():length():equals(0)} )}", attributes, true);
+ }
+
+ @Test
+ public void testReplaceNull() {
+ final Map<String, String> attributes = new HashMap<>();
+ verifyEquals("${attr:replaceNull('hello')}", attributes, "hello");
+ }
+
+ @Test
+ public void testReplace() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("attr", "hello");
+ verifyEquals("${attr:replace('hell', 'yell')}", attributes, "yello");
+ }
+
+ @Test
+ public void testReplaceAll() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("attr", "hello");
+ attributes.put("xyz", "00-00TEST.2014_01_01_000000_value");
+
+ verifyEquals("${xyz:replaceAll(\"^([^.]+)\\.([0-9]{4})_([0-9]{2})_([0-9]{2}).*$\", \"$3\")}", attributes, "01");
+ verifyEquals("${attr:replaceAll('l+', 'r')}", attributes, "hero");
+
+ attributes.clear();
+ attributes.put("filename1", "abc.gz");
+ attributes.put("filename2", "abc.g");
+ attributes.put("filename3", "abc.gz.gz");
+ attributes.put("filename4", "abc.gz.g");
+ attributes.put("abc", "hello world");
+
+ verifyEquals("${filename3:replaceAll('\\\\\\.gz$', '')}", attributes, "abc.gz.gz");
+ verifyEquals("${filename3:replaceAll('\\\\\\\\.gz$', '')}", attributes, "abc.gz.gz");
+ verifyEquals("${filename1:replaceAll('\\.gz$', '')}", attributes, "abc");
+ verifyEquals("${filename2:replaceAll('\\.gz$', '')}", attributes, "abc.g");
+ verifyEquals("${filename4:replaceAll('\\\\.gz$', '')}", attributes, "abc.gz.g");
+
+ verifyEquals("${abc:replaceAll( 'lo wor(ld)', '$0')}", attributes, "hello world");
+ verifyEquals("${abc:replaceAll( 'he(llo) world', '$1')}", attributes, "llo");
+ verifyEquals("${abc:replaceAll( 'xx', '$0')}", attributes, "hello world");
+ verifyEquals("${abc:replaceAll( '(xx)', '$1')}", attributes, "hello world");
+ verifyEquals("${abc:replaceAll( 'lo wor(ld)', '$1')}", attributes, "helld");
+
+ }
+
+ @Test
+ public void testReplaceAllWithOddNumberOfBackslashPairs() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("filename", "C:\\temp\\.txt");
+
+ verifyEquals("${filename:replace('\\\\', '/')}", attributes, "C:/temp/.txt");
+ verifyEquals("${filename:replaceAll('\\\\\\\\', '/')}", attributes, "C:/temp/.txt");
+ verifyEquals("${filename:replaceAll('\\\\\\.txt$', '')}", attributes, "C:\\temp");
+ }
+
+ @Test
+ public void testReplaceAllWithMatchingGroup() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("attr", "hello");
+
+ verifyEquals("${attr:replaceAll('.*?(l+).*', '$1')}", attributes, "ll");
+ }
+
+ @Test
+ public void testMathOperations() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("one", "1");
+ attributes.put("two", "2");
+ attributes.put("three", "3");
+ attributes.put("four", "4");
+ attributes.put("five", "5");
+ attributes.put("hundred", "100");
+
+ verifyEquals("${hundred:toNumber():multiply(2):divide(3):plus(1):mod(5)}", attributes, 2L);
+ }
+
+ @Test
+ public void testIndexOf() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("attr", "https://abc.go");
+ verifyEquals("${attr:indexOf('/')}", attributes, 6L);
+ }
+
+ @Test
+ public void testDate() {
+ final Calendar now = Calendar.getInstance();
+ final int year = now.get(Calendar.YEAR);
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("entryDate", String.valueOf(now.getTimeInMillis()));
+
+ verifyEquals("${entryDate:toNumber():toDate():format('yyyy')}", attributes, String.valueOf(year));
+
+ attributes.clear();
+ attributes.put("month", "3");
+ attributes.put("day", "4");
+ attributes.put("year", "2013");
+ assertEquals("63", Query.evaluateExpressions("${year:append('/'):append(${month}):append('/'):append(${day}):toDate('yyyy/MM/dd'):format('D')}", attributes, null));
+ assertEquals("63", Query.evaluateExpressions("${year:append('/'):append('${month}'):append('/'):append('${day}'):toDate('yyyy/MM/dd'):format('D')}", attributes, null));
+
+ verifyEquals("${year:append('/'):append(${month}):append('/'):append(${day}):toDate('yyyy/MM/dd'):format('D')}", attributes, "63");
+ }
+
+ @Test
+ public void testSystemProperty() {
+ System.setProperty("hello", "good-bye");
+ assertEquals("good-bye", Query.evaluateExpressions("${hello}"));
+ assertEquals("good-bye", Query.compile("${hello}").evaluate().getValue());
+ }
+
+ @Test
+ public void testAnyAttribute() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("abc", "zzz");
+ attributes.put("xyz", "abc");
+
+ verifyEquals("${anyAttribute('abc', 'xyz', 'missingAttr'):substring(1,2):equals('b')}", attributes, true);
+ verifyEquals("${anyAttribute('abc', 'xyz'):substring(1,2):equals('b')}", attributes, true);
+ verifyEquals("${anyAttribute('xyz', 'abc'):substring(1,2):equals('b')}", attributes, true);
+ verifyEquals("${anyAttribute('zz'):substring(1,2):equals('b')}", attributes, false);
+ verifyEquals("${anyAttribute('abc', 'zz'):isNull()}", attributes, true);
+ }
+
+ @Test
+ public void testAnyMatchingAttribute() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("abc", "zzz");
+ attributes.put("xyz", "abc");
+ attributes.put("123.cba", "hello");
+
+ verifyEquals("${anyMatchingAttribute('.{2}x', '.{2}z'):substring(1,2):equals('b')}", attributes, true);
+ verifyEquals("${anyMatchingAttribute('.*'):substring(1,2):equals('b')}", attributes, true);
+ verifyEquals("${anyMatchingAttribute('x{44}'):substring(1,2):equals('b')}", attributes, false);
+ verifyEquals("${anyMatchingAttribute('abc'):substring(1,2):equals('b')}", attributes, false);
+ verifyEquals("${anyMatchingAttribute('xyz'):substring(1,2):equals('b')}", attributes, true);
+ verifyEquals("${anyMatchingAttribute('xyz'):notNull()}", attributes, true);
+ verifyEquals("${anyMatchingAttribute('xyz'):isNull()}", attributes, false);
+ verifyEquals("${anyMatchingAttribute('xxxxxxxxx'):notNull()}", attributes, false);
+ verifyEquals("${anyMatchingAttribute('123\\.c.*'):matches('hello')}", attributes, true);
+ verifyEquals("${anyMatchingAttribute('123\\.c.*|a.c'):matches('zzz')}", attributes, true);
+ }
+
+ @Test
+ public void testAnyDelineatedValue() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("abc", "a,b,c");
+ attributes.put("xyz", "abc");
+
+ final String query = "${anyDelineatedValue('${abc}', ','):equals('b')}";
+ assertEquals(ResultType.BOOLEAN, Query.getResultType(query));
+
+ assertEquals("true", Query.evaluateExpressions(query, attributes, null));
+ assertEquals("true", Query.evaluateExpressions("${anyDelineatedValue('${abc}', ','):equals('a')}", attributes, null));
+ assertEquals("true", Query.evaluateExpressions("${anyDelineatedValue('${abc}', ','):equals('c')}", attributes, null));
+ assertEquals("false", Query.evaluateExpressions("${anyDelineatedValue('${abc}', ','):equals('d')}", attributes, null));
+
+ verifyEquals("${anyDelineatedValue(${abc}, ','):equals('b')}", attributes, true);
+ verifyEquals("${anyDelineatedValue(${abc}, ','):equals('a')}", attributes, true);
+ verifyEquals("${anyDelineatedValue(${abc}, ','):equals('c')}", attributes, true);
+ verifyEquals("${anyDelineatedValue(${abc}, ','):equals('d')}", attributes, false);
+ }
+
+ @Test
+ public void testAllDelineatedValues() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("abc", "a,b,c");
+ attributes.put("xyz", "abc");
+
+ final String query = "${allDelineatedValues('${abc}', ','):matches('[abc]')}";
+
+ assertEquals(ResultType.BOOLEAN, Query.getResultType(query));
+ assertEquals("true", Query.evaluateExpressions(query, attributes, null));
+ assertEquals("true", Query.evaluateExpressions(query, attributes, null));
+ assertEquals("false", Query.evaluateExpressions("${allDelineatedValues('${abc}', ','):matches('[abd]')}", attributes, null));
+ assertEquals("false", Query.evaluateExpressions("${allDelineatedValues('${abc}', ','):equals('a'):not()}", attributes, null));
+
+ verifyEquals("${allDelineatedValues(${abc}, ','):matches('[abc]')}", attributes, true);
+ verifyEquals("${allDelineatedValues(${abc}, ','):matches('[abd]')}", attributes, false);
+ verifyEquals("${allDelineatedValues(${abc}, ','):equals('a'):not()}", attributes, false);
+ }
+
+ @Test
+ public void testAllAttributes() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("abc", "1234");
+ attributes.put("xyz", "4132");
+ attributes.put("hello", "world!");
+
+ verifyEquals("${allAttributes('abc', 'xyz'):matches('\\d+')}", attributes, true);
+ verifyEquals("${allAttributes('abc', 'xyz'):toNumber():lt(99999)}", attributes, true);
+ verifyEquals("${allAttributes('abc', 'hello'):length():gt(3)}", attributes, true);
+ verifyEquals("${allAttributes('abc', 'hello'):length():equals(4)}", attributes, false);
+ verifyEquals("${allAttributes('abc', 'xyz'):length():equals(4)}", attributes, true);
+ verifyEquals("${allAttributes('abc', 'xyz', 'other'):isNull()}", attributes, false);
+
+ try {
+ Query.compile("${allAttributes('#ah'):equals('hello')");
+ Assert.fail("Was able to compile with allAttributes and an invalid attribute name");
+ } catch (final AttributeExpressionLanguageParsingException e) {
+ // expected behavior
+ }
+ }
+
+ @Test
+ public void testMathOperators() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("abc", "1234");
+ attributes.put("xyz", "4132");
+ attributes.put("hello", "world!");
+
+ verifyEquals("${xyz:toNumber():gt( ${abc:toNumber()} )}", attributes, true);
+ }
+
+ @Test
+ public void testAllMatchingAttributes() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("abc", "1234");
+ attributes.put("xyz", "4132");
+ attributes.put("hello", "world!");
+ attributes.put("123.cba", "hell.o");
+
+ System.out.println(printTree(Query.compile("${allMatchingAttributes('(abc|xyz)'):matches('\\\\d+')}").getTree()));
+
+ verifyEquals("${'123.cba':matches('hell\\.o')}", attributes, true);
+ verifyEquals("${allMatchingAttributes('123\\.cba'):equals('hell.o')}", attributes, true);
+ verifyEquals("${allMatchingAttributes('(abc|xyz)'):matches('\\d+')}", attributes, true);
+ verifyEquals("${allMatchingAttributes('[ax].*'):toNumber():lt(99999)}", attributes, true);
+ verifyEquals("${allMatchingAttributes('hell.'):length():gt(3)}", attributes, true);
+
+ verifyEquals("${allMatchingAttributes('123\\.cba'):equals('no')}", attributes, false);
+ }
+
+ @Test
+ public void testMatches() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("abc", "1234xyz4321");
+ attributes.put("end", "xyz");
+ attributes.put("xyz", "4132");
+ attributes.put("hello", "world!");
+ attributes.put("dotted", "abc.xyz");
+
+ final String evaluated = Query.evaluateExpressions("${abc:matches('1234${end}4321')}", attributes, null);
+ assertEquals("true", evaluated);
+
+ attributes.put("end", "888");
+ final String secondEvaluation = Query.evaluateExpressions("${abc:matches('1234${end}4321')}", attributes, null);
+ assertEquals("false", secondEvaluation);
+
+ verifyEquals("${dotted:matches('abc\\.xyz')}", attributes, true);
+ }
+
+ @Test
+ public void testFind() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("abc", "1234xyz4321");
+ attributes.put("end", "xyz");
+ attributes.put("xyz", "4132");
+ attributes.put("hello", "world!");
+ attributes.put("dotted", "abc.xyz");
+
+ final String evaluated = Query.evaluateExpressions("${abc:find('1234${end}4321')}", attributes, null);
+ assertEquals("true", evaluated);
+
+ attributes.put("end", "888");
+ final String secondEvaluation = Query.evaluateExpressions("${abc:find('${end}4321')}", attributes, null);
+ assertEquals("false", secondEvaluation);
+
+ verifyEquals("${dotted:find('\\.')}", attributes, true);
+ }
+
+ @Test
+ public void testSubstringAfter() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("filename", "file-255");
+
+ verifyEquals("${filename:substringAfter('')}", attributes, "file-255");
+ verifyEquals("${filename:substringAfterLast('')}", attributes, "file-255");
+ verifyEquals("${filename:substringBefore('')}", attributes, "file-255");
+ verifyEquals("${filename:substringBeforeLast('')}", attributes, "file-255");
+ verifyEquals("${filename:substringBefore('file')}", attributes, "");
+
+ attributes.put("uri", "sftp://some.uri");
+ verifyEquals("${uri:substringAfter('sftp')}", attributes, "://some.uri");
+ }
+
+ @Test
+ public void testSubstringAfterLast() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("filename", "file-file-255");
+
+ verifyEquals("${filename:substringAfterLast('file-')}", attributes, "255");
+ verifyEquals("${filename:substringAfterLast('5')}", attributes, "");
+ verifyEquals("${filename:substringAfterLast('x')}", attributes, "file-file-255");
+ }
+
+ @Test
+ public void testSubstringBefore() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("something", "some {} or other");
+
+ verifyEquals("${something:substringBefore('}')}", attributes, "some {");
+ }
+
+ @Test
+ public void testSubstring() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("filename", "file-255");
+
+ verifyEquals("${filename:substring(1, 2)}", attributes, "i");
+ verifyEquals("${filename:substring(4)}", attributes, "-255");
+ }
+
+ @Test
+ public void testToRadix() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("filename", "file-255");
+ attributes.put("filename2", "file-99999");
+
+ verifyEquals("${filename:substringAfter('-'):toNumber():toRadix(16):toUpper()}", attributes, "FF");
+ verifyEquals("${filename:substringAfter('-'):toNumber():toRadix(16, 4):toUpper()}", attributes, "00FF");
+ verifyEquals("${filename:substringAfter('-'):toNumber():toRadix(36, 3):toUpper()}", attributes, "073");
+ }
+
+ @Test
+ public void testDateFormatConversion() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("blue", "20130917162643");
+ verifyEquals("${blue:toDate('yyyyMMddHHmmss'):format(\"yyyy/MM/dd HH:mm:ss.SSS'Z'\")}", attributes, "2013/09/17 16:26:43.000Z");
+ }
+
+ @Test
+ public void testNot() {
+ verifyEquals("${ab:notNull():not()}", new HashMap<String, String>(), true);
+ }
+
+ @Test
+ public void testAttributesWithSpaces() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("ab", "abc");
+ attributes.put("a b", "abc");
+
+ verifyEquals("${ab}", attributes, "abc");
+ verifyEquals("${'a b'}", attributes, "abc");
+ verifyEquals("${'a b':replaceNull('')}", attributes, "");
+ }
+
+ @Test
+ public void testOr() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("filename1", "xabc");
+ attributes.put("filename2", "yabc");
+ attributes.put("filename3", "abcxy");
+
+ verifyEquals("${filename1:startsWith('x'):or(true)}", attributes, true);
+ verifyEquals("${filename1:startsWith('x'):or( ${filename1:startsWith('y')} )}", attributes, true);
+ verifyEquals("${filename2:startsWith('x'):or( ${filename2:startsWith('y')} )}", attributes, true);
+ verifyEquals("${filename3:startsWith('x'):or( ${filename3:startsWith('y')} )}", attributes, false);
+ verifyEquals("${filename1:startsWith('x'):or( ${filename2:startsWith('y')} )}", attributes, true);
+ verifyEquals("${filename2:startsWith('x'):or( ${filename1:startsWith('y')} )}", attributes, false);
+ }
+
+ @Test
+ public void testAnd() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("filename1", "xabc");
+ attributes.put("filename2", "yabc");
+ attributes.put("filename 3", "abcxy");
+
+ verifyEquals("${filename1:startsWith('x'):and(true)}", attributes, true);
+ verifyEquals("${filename1:startsWith('x') : and( false )}", attributes, false);
+ verifyEquals("${filename1:startsWith('x'):and( ${filename1:startsWith('y')} )}", attributes, false);
+ verifyEquals("${filename2:startsWith('x'):and( ${filename2:startsWith('y')} )}", attributes, false);
+ verifyEquals("${filename3:startsWith('x'):and( ${filename3:startsWith('y')} )}", attributes, false);
+ verifyEquals("${filename1:startsWith('x'):and( ${filename2:startsWith('y')} )}", attributes, true);
+ verifyEquals("${filename2:startsWith('x'):and( ${filename1:startsWith('y')} )}", attributes, false);
+ verifyEquals("${filename1:startsWith('x'):and( ${'filename 3':endsWith('y')} )}", attributes, true);
+ }
+
+ @Test
+ public void testAndOrNot() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("filename1", "xabc");
+ attributes.put("filename2", "yabc");
+ attributes.put("filename 3", "abcxy");
+
+ final String query
+ = "${"
+ + " 'non-existing':notNull():not():and(" + // true AND (
+ " ${filename1:startsWith('y')" + // false
+ " :or(" + // or
+ " ${ filename1:startsWith('x'):and(false) }" + // false
+ " ):or(" + // or
+ " ${ filename2:endsWith('xxxx'):or( ${'filename 3':length():gt(1)} ) }" + // true )
+ " )}"
+ + " )"
+ + "}";
+
+ System.out.println(query);
+ verifyEquals(query, attributes, true);
+ }
+
+ @Test
+ public void testAndOrLogicWithAnyAll() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("filename1", "xabc");
+ attributes.put("filename2", "yabc");
+ attributes.put("filename 3", "abcxy");
+
+ verifyEquals("${anyMatchingAttribute('filename.*'):contains('abc'):and( ${filename2:equals('yabc')} )}", attributes, true);
+ verifyEquals("${anyMatchingAttribute('filename.*'):contains('abc'):and( ${filename2:equals('xabc')} )}", attributes, false);
+ verifyEquals("${anyMatchingAttribute('filename.*'):contains('abc'):not():or( ${filename2:equals('yabc')} )}", attributes, true);
+ verifyEquals("${anyMatchingAttribute('filename.*'):contains('abc'):not():or( ${filename2:equals('xabc')} )}", attributes, false);
+ }
+
+ @Test
+ public void testKeywords() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("UUID", "123");
+ verifyEquals("${ 'UUID':toNumber():equals(123) }", attributes, true);
+ }
+
+ @Test
+ public void testEqualsNumber() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("abc", "123");
+ verifyEquals("${ abc:toNumber():equals(123) }", attributes, true);
+ }
+
+ @Test
+ public void testSubjectAsEmbeddedExpressionWithSurroundChars() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("b", "x");
+ attributes.put("abcxcba", "hello");
+
+ final String evaluated = Query.evaluateExpressions("${ 'abc${b}cba':substring(0, 1) }", attributes, null);
+ assertEquals("h", evaluated);
+ }
+
+ @Test
+ public void testToNumberFunctionReturnsNumberType() {
+ assertEquals(ResultType.NUMBER, Query.getResultType("${header.size:toNumber()}"));
+ }
+
+ @Test
+ public void testAnyAttributeEmbedded() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("a1", "test1");
+ attributes.put("b2", "2test");
+ attributes.put("c3", "3test3");
+
+ final String query = "${a1:equals('test1'):and( ${anyAttribute('a1','b2','c3'):contains('2')})}";
+ verifyEquals(query, attributes, true);
+ }
+
+ @Test
+ public void testEvaluateWithinCurlyBraces() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("abc", "xyz");
+
+ final String query = "{ ${abc} }";
+ final List<String> expressions = Query.extractExpressions(query);
+ assertEquals(1, expressions.size());
+ assertEquals("${abc}", expressions.get(0));
+ assertEquals("{ xyz }", Query.evaluateExpressions(query, attributes));
+ }
+
+ @Test
+ public void testLiteralFunction() {
+ final Map<String, String> attrs = Collections.<String, String> emptyMap();
+ verifyEquals("${literal(2):gt(1)}", attrs, true);
+ verifyEquals("${literal('hello'):substring(0, 1):equals('h')}", attrs, true);
+ }
+
+ @Test
+ public void testFunctionAfterReduce() {
+ // Cannot call gt(2) after count() because count() is a 'reducing function'
+ // and must be the last function in an expression.
+ assertFalse(Query.isValidExpression("${allMatchingAttributes('a.*'):contains('2'):count():gt(2)}"));
+
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("a.1", "245");
+ attributes.put("a.2", "123");
+ attributes.put("a.3", "732");
+ attributes.put("a.4", "343");
+ attributes.put("a.5", "553");
+
+ final String endsWithCount = "${allMatchingAttributes('a.*'):contains('2'):count()}";
+ assertTrue(Query.isValidExpression(endsWithCount));
+ verifyEquals(endsWithCount, attributes, 3L);
+
+ // in order to check if value is greater than 2, need to first evaluate the
+ // 'aggregate' and 'reducing' functions as an inner expression. Then we can
+ // use the literal() function to make the result of the inner expression the subject
+ // of the function gt()
+ final String usingLiteral = "${literal(" + endsWithCount + "):gt(2)}";
+ assertTrue(Query.isValidExpression(usingLiteral));
+ verifyEquals(usingLiteral, attributes, true);
+
+ attributes.clear();
+ attributes.put("a1", "123");
+ attributes.put("a2", "321");
+ verifyEquals("${allMatchingAttributes('a.*'):contains('2')}", attributes, true);
+ verifyEquals("${allMatchingAttributes('a.*'):contains('2'):toUpper():equals('TRUE')}", attributes, true);
+ verifyEquals("${allMatchingAttributes('a.*'):contains('2'):equals('true'):and( ${literal(true)} )}", attributes, true);
+ }
+
+ private void verifyEquals(final String expression, final Map<String, String> attributes, final Object expectedResult) {
+ Query.validateExpression(expression, false);
+ assertEquals(String.valueOf(expectedResult), Query.evaluateExpressions(expression, attributes, null));
+
+ final Query query = Query.compile(expression);
+ final QueryResult<?> result = query.evaluate(attributes);
+
+ if (expectedResult instanceof Number) {
+ assertEquals(ResultType.NUMBER, result.getResultType());
+ } else if (expectedResult instanceof Boolean) {
+ assertEquals(ResultType.BOOLEAN, result.getResultType());
+ } else {
+ assertEquals(ResultType.STRING, result.getResultType());
+ }
+
+ assertEquals(expectedResult, result.getValue());
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestStandardPreparedQuery.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestStandardPreparedQuery.java b/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestStandardPreparedQuery.java
new file mode 100644
index 0000000..5acba8d
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestStandardPreparedQuery.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class TestStandardPreparedQuery {
+
+ @Test
+ public void testSimpleReference() {
+ final Map<String, String> attrs = new HashMap<>();
+ attrs.put("xx", "world");
+
+ assertEquals("world", evaluate("${xx}", attrs));
+ assertEquals("hello, world!", evaluate("hello, ${xx}!", attrs));
+ }
+
+ @Test
+ public void testEmbeddedReference() {
+ final Map<String, String> attrs = new HashMap<>();
+ attrs.put("xx", "yy");
+ attrs.put("yy", "world");
+
+ assertEquals("world", evaluate("${${xx}}", attrs));
+ }
+
+ @Test
+ public void test10MIterations() {
+ final Map<String, String> attrs = new HashMap<>();
+ attrs.put("xx", "world");
+
+ final StandardPreparedQuery prepared = (StandardPreparedQuery) Query.prepare("${xx}");
+ final long start = System.nanoTime();
+ for (int i = 0; i < 10000000; i++) {
+ assertEquals("world", prepared.evaluateExpressions(attrs, null));
+ }
+ final long nanos = System.nanoTime() - start;
+ System.out.println(TimeUnit.NANOSECONDS.toMillis(nanos));
+ }
+
+ @Test
+ @Ignore("Takes too long")
+ public void test10MIterationsWithQuery() {
+ final Map<String, String> attrs = new HashMap<>();
+ attrs.put("xx", "world");
+
+ final long start = System.nanoTime();
+ for (int i = 0; i < 10000000; i++) {
+ assertEquals("world", Query.evaluateExpressions("${xx}", attrs));
+ }
+ final long nanos = System.nanoTime() - start;
+ System.out.println(TimeUnit.NANOSECONDS.toMillis(nanos));
+
+ }
+
+ @Test
+ public void testSeveralSequentialExpressions() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("audience", "World");
+ attributes.put("comma", ",");
+ attributes.put("question", " how are you?");
+ assertEquals("Hello, World, how are you?!", evaluate("Hello, ${audience}${comma}${question}!", attributes));
+
+ }
+
+ private String evaluate(final String query, final Map<String, String> attrs) {
+ final String evaluated = ((StandardPreparedQuery) Query.prepare(query)).evaluateExpressions(attrs, null);
+ return evaluated;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-flowfile-packager/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-flowfile-packager/pom.xml b/nifi-commons/nifi-flowfile-packager/pom.xml
new file mode 100644
index 0000000..7173f03
--- /dev/null
+++ b/nifi-commons/nifi-flowfile-packager/pom.xml
@@ -0,0 +1,34 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-commons</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>nifi-flowfile-packager</artifactId>
+ <packaging>jar</packaging>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-flowfile-packager/src/main/java/org/apache/nifi/util/FlowFilePackager.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-flowfile-packager/src/main/java/org/apache/nifi/util/FlowFilePackager.java b/nifi-commons/nifi-flowfile-packager/src/main/java/org/apache/nifi/util/FlowFilePackager.java
new file mode 100644
index 0000000..ae16f99
--- /dev/null
+++ b/nifi-commons/nifi-flowfile-packager/src/main/java/org/apache/nifi/util/FlowFilePackager.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Map;
+
+public interface FlowFilePackager {
+
+ void packageFlowFile(InputStream in, OutputStream out, Map<String, String> attributes, long fileSize) throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-flowfile-packager/src/main/java/org/apache/nifi/util/FlowFilePackagerV1.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-flowfile-packager/src/main/java/org/apache/nifi/util/FlowFilePackagerV1.java b/nifi-commons/nifi-flowfile-packager/src/main/java/org/apache/nifi/util/FlowFilePackagerV1.java
new file mode 100644
index 0000000..479ac58
--- /dev/null
+++ b/nifi-commons/nifi-flowfile-packager/src/main/java/org/apache/nifi/util/FlowFilePackagerV1.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.lang3.StringEscapeUtils;
+
+public class FlowFilePackagerV1 implements FlowFilePackager {
+
+ public static final String FILENAME_ATTRIBUTES = "flowfile.attributes";
+ public static final String FILENAME_CONTENT = "flowfile.content";
+ public static final int DEFAULT_TAR_PERMISSIONS = 0644;
+
+ private final int tarPermissions;
+
+ public FlowFilePackagerV1() {
+ this(DEFAULT_TAR_PERMISSIONS);
+ }
+
+ public FlowFilePackagerV1(final int tarPermissions) {
+ this.tarPermissions = tarPermissions;
+ }
+
+ @Override
+ public void packageFlowFile(final InputStream in, final OutputStream out, final Map<String, String> attributes, final long fileSize) throws IOException {
+ try (final TarArchiveOutputStream tout = new TarArchiveOutputStream(out)) {
+ writeAttributesEntry(attributes, tout);
+ writeContentEntry(tout, in, fileSize);
+ tout.finish();
+ tout.flush();
+ tout.close();
+ }
+ }
+
+ private void writeAttributesEntry(final Map<String, String> attributes, final TarArchiveOutputStream tout) throws IOException {
+ final StringBuilder sb = new StringBuilder();
+ sb.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?><!DOCTYPE properties\n SYSTEM \"http://java.sun.com/dtd/properties.dtd\">\n");
+ sb.append("<properties>");
+ for (final Map.Entry<String, String> entry : attributes.entrySet()) {
+ final String escapedKey = StringEscapeUtils.escapeXml11(entry.getKey());
+ final String escapedValue = StringEscapeUtils.escapeXml11(entry.getValue());
+ sb.append("\n <entry key=\"").append(escapedKey).append("\">").append(escapedValue).append("</entry>");
+ }
+ sb.append("</properties>");
+
+ final byte[] metaBytes = sb.toString().getBytes(StandardCharsets.UTF_8);
+ final TarArchiveEntry attribEntry = new TarArchiveEntry(FILENAME_ATTRIBUTES);
+ attribEntry.setMode(tarPermissions);
+ attribEntry.setSize(metaBytes.length);
+ tout.putArchiveEntry(attribEntry);
+ tout.write(metaBytes);
+ tout.closeArchiveEntry();
+ }
+
+ private void writeContentEntry(final TarArchiveOutputStream tarOut, final InputStream inStream, final long fileSize) throws IOException {
+ final TarArchiveEntry entry = new TarArchiveEntry(FILENAME_CONTENT);
+ entry.setMode(tarPermissions);
+ entry.setSize(fileSize);
+ tarOut.putArchiveEntry(entry);
+ final byte[] buffer = new byte[512 << 10];//512KB
+ int bytesRead = 0;
+ while ((bytesRead = inStream.read(buffer)) != -1) { //still more data to read
+ if (bytesRead > 0) {
+ tarOut.write(buffer, 0, bytesRead);
+ }
+ }
+
+ copy(inStream, tarOut);
+ tarOut.closeArchiveEntry();
+ }
+
+ public static long copy(final InputStream source, final OutputStream destination) throws IOException {
+ final byte[] buffer = new byte[8192];
+ int len;
+ long totalCount = 0L;
+ while ((len = source.read(buffer)) > 0) {
+ destination.write(buffer, 0, len);
+ totalCount += len;
+ }
+ return totalCount;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-flowfile-packager/src/main/java/org/apache/nifi/util/FlowFilePackagerV2.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-flowfile-packager/src/main/java/org/apache/nifi/util/FlowFilePackagerV2.java b/nifi-commons/nifi-flowfile-packager/src/main/java/org/apache/nifi/util/FlowFilePackagerV2.java
new file mode 100644
index 0000000..6f9d6b1
--- /dev/null
+++ b/nifi-commons/nifi-flowfile-packager/src/main/java/org/apache/nifi/util/FlowFilePackagerV2.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Map;
+
+/**
+ * <p>
+ * Packages a FlowFile, including both its content and its attributes into a
+ * single file that is stream-friendly. The encoding scheme is as such:
+ * </p>
+ *
+ * <pre>
+ * Length Field : indicates the number of Flow File Attributes in the stream
+ * 1 to N times (N=number of Flow File Attributes):
+ * String Field : Flow File Attribute key name
+ * String Field : Flow File Attribute value
+ * Long : 8 bytes indicating the length of the Flow File content
+ * Content : The next M bytes are the content of the Flow File.
+ * </pre>
+ *
+ * <pre>
+ * Encoding of String Field is as follows:
+ * Length Field : indicates the length of the String
+ * 1 to N bytes (N=String length, determined by previous field, as described above) : The UTF-8 encoded string value.
+ * </pre>
+ *
+ * <pre>
+ * Encoding of Length Field is as follows:
+ * First 2 bytes: Indicate length. If both bytes = 255, this is a special value indicating that the length is
+ * greater than or equal to 65536 bytes; therefore, the next 4 bytes will indicate the actual length.
+ * </pre>
+ *
+ * <p>
+ * Note: All byte-order encoding is Network Byte Order (Most Significant Byte
+ * first)
+ * </p>
+ *
+ * <p>
+ * The following example shows the bytes expected if we were to encode a
+ * FlowFile containing the following attributes where the content is the text
+ * "Hello World!":
+ *
+ * <br><br>
+ * Attributes:
+ * <pre>
+ * +-------+-------+
+ * | Key + Value |
+ * + --------------+
+ * | A | a |
+ * + --------------+
+ * | B | b |
+ * + --------------+
+ * </pre> Content:<br>
+ * Hello World!
+ * <br><br>
+ * Packaged Byte Encoding (In Hexadecimal Form):
+ * <p>
+ *
+ * <pre>
+ * 00 02 00 01 41 00 01 61
+ * 00 01 42 00 01 62 00 00
+ * 00 00 00 00 00 0C 48 65
+ * 6C 6C 6F 20 57 6F 72 6C
+ * 64 21
+ * </pre>
+ */
+public class FlowFilePackagerV2 implements FlowFilePackager {
+
+ private static final int MAX_VALUE_2_BYTES = 65535;
+ private final byte[] writeBuffer = new byte[8];
+
+ @Override
+ public void packageFlowFile(final InputStream in, final OutputStream out, final Map<String, String> attributes, final long fileSize) throws IOException {
+ writeFieldLength(out, attributes.size()); //write out the number of attributes
+ for (final Map.Entry<String, String> entry : attributes.entrySet()) { //write out each attribute key/value pair
+ writeString(entry.getKey(), out);
+ writeString(entry.getValue(), out);
+ }
+ writeLong(out, fileSize);//write out length of data
+ copy(in, out);//write out the actual flow file payload
+ }
+
+ private void copy(final InputStream in, final OutputStream out) throws IOException {
+ final byte[] buffer = new byte[65536];
+ int len;
+ while ((len = in.read(buffer)) > 0) {
+ out.write(buffer, 0, len);
+ }
+ }
+
+ private void writeString(final String val, final OutputStream out) throws IOException {
+ final byte[] bytes = val.getBytes("UTF-8");
+ writeFieldLength(out, bytes.length);
+ out.write(bytes);
+ }
+
+ private void writeFieldLength(final OutputStream out, final int numBytes) throws IOException {
+ // If the value is less than the max value that can be fit into 2 bytes, just use the
+ // actual value. Otherwise, we will set the first 2 bytes to 255/255 and then use the next
+ // 4 bytes to indicate the real length.
+ if (numBytes < MAX_VALUE_2_BYTES) {
+ writeBuffer[0] = (byte) (numBytes >>> 8);
+ writeBuffer[1] = (byte) (numBytes);
+ out.write(writeBuffer, 0, 2);
+ } else {
+ writeBuffer[0] = (byte) 0xff;
+ writeBuffer[1] = (byte) 0xff;
+ writeBuffer[2] = (byte) (numBytes >>> 24);
+ writeBuffer[3] = (byte) (numBytes >>> 16);
+ writeBuffer[4] = (byte) (numBytes >>> 8);
+ writeBuffer[5] = (byte) (numBytes);
+ out.write(writeBuffer, 0, 6);
+ }
+ }
+
+ private void writeLong(final OutputStream out, final long val) throws IOException {
+ writeBuffer[0] = (byte) (val >>> 56);
+ writeBuffer[1] = (byte) (val >>> 48);
+ writeBuffer[2] = (byte) (val >>> 40);
+ writeBuffer[3] = (byte) (val >>> 32);
+ writeBuffer[4] = (byte) (val >>> 24);
+ writeBuffer[5] = (byte) (val >>> 16);
+ writeBuffer[6] = (byte) (val >>> 8);
+ writeBuffer[7] = (byte) (val);
+ out.write(writeBuffer, 0, 8);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-flowfile-packager/src/main/java/org/apache/nifi/util/FlowFilePackagerV3.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-flowfile-packager/src/main/java/org/apache/nifi/util/FlowFilePackagerV3.java b/nifi-commons/nifi-flowfile-packager/src/main/java/org/apache/nifi/util/FlowFilePackagerV3.java
new file mode 100644
index 0000000..181f3e3
--- /dev/null
+++ b/nifi-commons/nifi-flowfile-packager/src/main/java/org/apache/nifi/util/FlowFilePackagerV3.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Map;
+
+public class FlowFilePackagerV3 implements FlowFilePackager {
+
+ public static final byte[] MAGIC_HEADER = {'N', 'i', 'F', 'i', 'F', 'F', '3'};
+ private static final int MAX_VALUE_2_BYTES = 65535;
+ private final byte[] writeBuffer = new byte[8];
+
+ @Override
+ public void packageFlowFile(final InputStream in, final OutputStream out, final Map<String, String> attributes, final long fileSize) throws IOException {
+ out.write(MAGIC_HEADER);
+
+ if (attributes == null) {
+ writeFieldLength(out, 0);
+ } else {
+ writeFieldLength(out, attributes.size()); //write out the number of attributes
+ for (final Map.Entry<String, String> entry : attributes.entrySet()) { //write out each attribute key/value pair
+ writeString(entry.getKey(), out);
+ writeString(entry.getValue(), out);
+ }
+ }
+
+ writeLong(out, fileSize);//write out length of data
+ copy(in, out);//write out the actual flow file payload
+ }
+
+ private void copy(final InputStream in, final OutputStream out) throws IOException {
+ final byte[] buffer = new byte[65536];
+ int len;
+ while ((len = in.read(buffer)) > 0) {
+ out.write(buffer, 0, len);
+ }
+ }
+
+ private void writeString(final String val, final OutputStream out) throws IOException {
+ final byte[] bytes = val.getBytes("UTF-8");
+ writeFieldLength(out, bytes.length);
+ out.write(bytes);
+ }
+
+ private void writeFieldLength(final OutputStream out, final int numBytes) throws IOException {
+ // If the value is less than the max value that can be fit into 2 bytes, just use the
+ // actual value. Otherwise, we will set the first 2 bytes to 255/255 and then use the next
+ // 4 bytes to indicate the real length.
+ if (numBytes < MAX_VALUE_2_BYTES) {
+ writeBuffer[0] = (byte) (numBytes >>> 8);
+ writeBuffer[1] = (byte) (numBytes);
+ out.write(writeBuffer, 0, 2);
+ } else {
+ writeBuffer[0] = (byte) 0xff;
+ writeBuffer[1] = (byte) 0xff;
+ writeBuffer[2] = (byte) (numBytes >>> 24);
+ writeBuffer[3] = (byte) (numBytes >>> 16);
+ writeBuffer[4] = (byte) (numBytes >>> 8);
+ writeBuffer[5] = (byte) (numBytes);
+ out.write(writeBuffer, 0, 6);
+ }
+ }
+
+ private void writeLong(final OutputStream out, final long val) throws IOException {
+ writeBuffer[0] = (byte) (val >>> 56);
+ writeBuffer[1] = (byte) (val >>> 48);
+ writeBuffer[2] = (byte) (val >>> 40);
+ writeBuffer[3] = (byte) (val >>> 32);
+ writeBuffer[4] = (byte) (val >>> 24);
+ writeBuffer[5] = (byte) (val >>> 16);
+ writeBuffer[6] = (byte) (val >>> 8);
+ writeBuffer[7] = (byte) (val);
+ out.write(writeBuffer, 0, 8);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-flowfile-packager/src/main/java/org/apache/nifi/util/FlowFileUnpackager.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-flowfile-packager/src/main/java/org/apache/nifi/util/FlowFileUnpackager.java b/nifi-commons/nifi-flowfile-packager/src/main/java/org/apache/nifi/util/FlowFileUnpackager.java
new file mode 100644
index 0000000..fd9d92d
--- /dev/null
+++ b/nifi-commons/nifi-flowfile-packager/src/main/java/org/apache/nifi/util/FlowFileUnpackager.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Map;
+
+public interface FlowFileUnpackager {
+
+ Map<String, String> unpackageFlowFile(InputStream in, OutputStream out) throws IOException;
+
+ boolean hasMoreData() throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-flowfile-packager/src/main/java/org/apache/nifi/util/FlowFileUnpackagerV1.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-flowfile-packager/src/main/java/org/apache/nifi/util/FlowFileUnpackagerV1.java b/nifi-commons/nifi-flowfile-packager/src/main/java/org/apache/nifi/util/FlowFileUnpackagerV1.java
new file mode 100644
index 0000000..b96534a
--- /dev/null
+++ b/nifi-commons/nifi-flowfile-packager/src/main/java/org/apache/nifi/util/FlowFileUnpackagerV1.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+
+public class FlowFileUnpackagerV1 implements FlowFileUnpackager {
+
+ private int flowFilesRead = 0;
+
+ @Override
+ public Map<String, String> unpackageFlowFile(final InputStream in, final OutputStream out) throws IOException {
+ flowFilesRead++;
+ final TarArchiveInputStream tarIn = new TarArchiveInputStream(in);
+ final TarArchiveEntry attribEntry = tarIn.getNextTarEntry();
+ if (attribEntry == null) {
+ return null;
+ }
+
+ final Map<String, String> attributes;
+ if (attribEntry.getName().equals(FlowFilePackagerV1.FILENAME_ATTRIBUTES)) {
+ attributes = getAttributes(tarIn);
+ } else {
+ throw new IOException("Expected two tar entries: "
+ + FlowFilePackagerV1.FILENAME_CONTENT + " and "
+ + FlowFilePackagerV1.FILENAME_ATTRIBUTES);
+ }
+
+ final TarArchiveEntry contentEntry = tarIn.getNextTarEntry();
+
+ if (contentEntry != null && contentEntry.getName().equals(FlowFilePackagerV1.FILENAME_CONTENT)) {
+ final byte[] buffer = new byte[512 << 10];//512KB
+ int bytesRead = 0;
+ while ((bytesRead = tarIn.read(buffer)) != -1) { //still more data to read
+ if (bytesRead > 0) {
+ out.write(buffer, 0, bytesRead);
+ }
+ }
+ out.flush();
+ } else {
+ throw new IOException("Expected two tar entries: "
+ + FlowFilePackagerV1.FILENAME_CONTENT + " and "
+ + FlowFilePackagerV1.FILENAME_ATTRIBUTES);
+ }
+
+ return attributes;
+ }
+
+ protected Map<String, String> getAttributes(final TarArchiveInputStream stream) throws IOException {
+
+ final Properties props = new Properties();
+ props.loadFromXML(new NonCloseableInputStream(stream));
+
+ final Map<String, String> result = new HashMap<>();
+ for (final Entry<Object, Object> entry : props.entrySet()) {
+ final Object keyObject = entry.getKey();
+ final Object valueObject = entry.getValue();
+ if (!(keyObject instanceof String)) {
+ throw new IOException("Flow file attributes object contains key of type "
+ + keyObject.getClass().getCanonicalName()
+ + " but expected java.lang.String");
+ } else if (!(keyObject instanceof String)) {
+ throw new IOException("Flow file attributes object contains value of type "
+ + keyObject.getClass().getCanonicalName()
+ + " but expected java.lang.String");
+ }
+
+ final String key = (String) keyObject;
+ final String value = (String) valueObject;
+ result.put(key, value);
+ }
+
+ return result;
+ }
+
+ @Override
+ public boolean hasMoreData() throws IOException {
+ return flowFilesRead == 0;
+ }
+
+ public static final class NonCloseableInputStream extends InputStream {
+
+ final InputStream stream;
+
+ public NonCloseableInputStream(final InputStream stream) {
+ this.stream = stream;
+ }
+
+ @Override
+ public void close() {
+ }
+
+ @Override
+ public int read() throws IOException {
+ return stream.read();
+ }
+
+ @Override
+ public int available() throws IOException {
+ return stream.available();
+ }
+
+ @Override
+ public synchronized void mark(int readlimit) {
+ stream.mark(readlimit);
+ }
+
+ @Override
+ public synchronized void reset() throws IOException {
+ stream.reset();
+ }
+
+ @Override
+ public boolean markSupported() {
+ return stream.markSupported();
+ }
+
+ @Override
+ public long skip(long n) throws IOException {
+ return stream.skip(n);
+ }
+
+ @Override
+ public int read(byte b[], int off, int len) throws IOException {
+ return stream.read(b, off, len);
+ }
+
+ @Override
+ public int read(byte b[]) throws IOException {
+ return stream.read(b);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-flowfile-packager/src/main/java/org/apache/nifi/util/FlowFileUnpackagerV2.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-flowfile-packager/src/main/java/org/apache/nifi/util/FlowFileUnpackagerV2.java b/nifi-commons/nifi-flowfile-packager/src/main/java/org/apache/nifi/util/FlowFileUnpackagerV2.java
new file mode 100644
index 0000000..500015f
--- /dev/null
+++ b/nifi-commons/nifi-flowfile-packager/src/main/java/org/apache/nifi/util/FlowFileUnpackagerV2.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Map;
+
+public class FlowFileUnpackagerV2 implements FlowFileUnpackager {
+
+ private final byte readBuffer[] = new byte[8192];
+ private Map<String, String> nextAttributes = null;
+ private boolean haveReadSomething = false;
+
+ @Override
+ public boolean hasMoreData() throws IOException {
+ return nextAttributes != null || !haveReadSomething;
+ }
+
+ protected Map<String, String> readAttributes(final InputStream in) throws IOException {
+ final Map<String, String> attributes = new HashMap<>();
+ final Integer numAttributes = readFieldLength(in); //read number of attributes
+ if (numAttributes == null) {
+ return null;
+ }
+ if (numAttributes == 0) {
+ throw new IOException("flow files cannot have zero attributes");
+ }
+ for (int i = 0; i < numAttributes; i++) { //read each attribute key/value pair
+ final String key = readString(in);
+ final String value = readString(in);
+ attributes.put(key, value);
+ }
+
+ return attributes;
+ }
+
+ @Override
+ public Map<String, String> unpackageFlowFile(final InputStream in, final OutputStream out) throws IOException {
+ final Map<String, String> attributes;
+ if (nextAttributes != null) {
+ attributes = nextAttributes;
+ } else {
+ attributes = readAttributes(in);
+ }
+
+ final long expectedNumBytes = readLong(in); // read length of payload
+ copy(in, out, expectedNumBytes); // read payload
+
+ nextAttributes = readAttributes(in);
+ haveReadSomething = true;
+
+ return attributes;
+ }
+
+ protected String readString(final InputStream in) throws IOException {
+ final Integer numBytes = readFieldLength(in);
+ if (numBytes == null) {
+ throw new EOFException();
+ }
+ final byte[] bytes = new byte[numBytes];
+ fillBuffer(in, bytes, numBytes);
+ return new String(bytes, "UTF-8");
+ }
+
+ private void fillBuffer(final InputStream in, final byte[] buffer, final int length) throws IOException {
+ int bytesRead;
+ int totalBytesRead = 0;
+ while ((bytesRead = in.read(buffer, totalBytesRead, length - totalBytesRead)) > 0) {
+ totalBytesRead += bytesRead;
+ }
+ if (totalBytesRead != length) {
+ throw new EOFException();
+ }
+ }
+
+ protected long copy(final InputStream in, final OutputStream out, final long numBytes) throws IOException {
+ int bytesRead;
+ long totalBytesRead = 0L;
+ while ((bytesRead = in.read(readBuffer, 0, (int) Math.min(readBuffer.length, numBytes - totalBytesRead))) > 0) {
+ out.write(readBuffer, 0, bytesRead);
+ totalBytesRead += bytesRead;
+ }
+
+ if (totalBytesRead < numBytes) {
+ throw new EOFException("Expected " + numBytes + " but received " + totalBytesRead);
+ }
+
+ return totalBytesRead;
+ }
+
+ protected long readLong(final InputStream in) throws IOException {
+ fillBuffer(in, readBuffer, 8);
+ return (((long) readBuffer[0] << 56)
+ + ((long) (readBuffer[1] & 255) << 48)
+ + ((long) (readBuffer[2] & 255) << 40)
+ + ((long) (readBuffer[3] & 255) << 32)
+ + ((long) (readBuffer[4] & 255) << 24)
+ + ((readBuffer[5] & 255) << 16)
+ + ((readBuffer[6] & 255) << 8)
+ + ((readBuffer[7] & 255)));
+ }
+
+ private Integer readFieldLength(final InputStream in) throws IOException {
+ final int firstValue = in.read();
+ final int secondValue = in.read();
+ if (firstValue < 0) {
+ return null;
+ }
+ if (secondValue < 0) {
+ throw new EOFException();
+ }
+ if (firstValue == 0xff && secondValue == 0xff) {
+ int ch1 = in.read();
+ int ch2 = in.read();
+ int ch3 = in.read();
+ int ch4 = in.read();
+ if ((ch1 | ch2 | ch3 | ch4) < 0) {
+ throw new EOFException();
+ }
+ return ((ch1 << 24) + (ch2 << 16) + (ch3 << 8) + (ch4));
+ } else {
+ return ((firstValue << 8) + (secondValue));
+ }
+ }
+}
[29/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/SslContextFactory.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/SslContextFactory.java b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/SslContextFactory.java
new file mode 100644
index 0000000..656573d
--- /dev/null
+++ b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/SslContextFactory.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.security.util;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.KeyManagementException;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.SecureRandom;
+import java.security.UnrecoverableKeyException;
+import java.security.cert.CertificateException;
+
+import javax.net.ssl.KeyManager;
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.TrustManagerFactory;
+
+/**
+ * A factory for creating SSL contexts using the application's security
+ * properties.
+ *
+ */
+public final class SslContextFactory {
+
+ public static enum ClientAuth {
+
+ WANT,
+ REQUIRED,
+ NONE
+ }
+
+ /**
+ * Creates a SSLContext instance using the given information.
+ *
+ * @param keystore the full path to the keystore
+ * @param keystorePasswd the keystore password
+ * @param keystoreType the type of keystore (e.g., PKCS12, JKS)
+ * @param truststore the full path to the truststore
+ * @param truststorePasswd the truststore password
+ * @param truststoreType the type of truststore (e.g., PKCS12, JKS)
+ * @param clientAuth the type of client authentication
+ * @param protocol the protocol to use for the SSL connection
+ *
+ * @return a SSLContext instance
+ * @throws java.security.KeyStoreException if any issues accessing the keystore
+ * @throws java.io.IOException for any problems loading the keystores
+ * @throws java.security.NoSuchAlgorithmException if an algorithm is found to be used but is unknown
+ * @throws java.security.cert.CertificateException if there is an issue with the certificate
+ * @throws java.security.UnrecoverableKeyException if the key is insufficient
+ * @throws java.security.KeyManagementException if unable to manage the key
+ */
+ public static SSLContext createSslContext(
+ final String keystore, final char[] keystorePasswd, final String keystoreType,
+ final String truststore, final char[] truststorePasswd, final String truststoreType,
+ final ClientAuth clientAuth, final String protocol)
+ throws KeyStoreException, IOException, NoSuchAlgorithmException, CertificateException,
+ UnrecoverableKeyException, KeyManagementException {
+
+ // prepare the keystore
+ final KeyStore keyStore = KeyStore.getInstance(keystoreType);
+ try (final InputStream keyStoreStream = new FileInputStream(keystore)) {
+ keyStore.load(keyStoreStream, keystorePasswd);
+ }
+ final KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
+ keyManagerFactory.init(keyStore, keystorePasswd);
+
+ // prepare the truststore
+ final KeyStore trustStore = KeyStore.getInstance(truststoreType);
+ try (final InputStream trustStoreStream = new FileInputStream(truststore)) {
+ trustStore.load(trustStoreStream, truststorePasswd);
+ }
+ final TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
+ trustManagerFactory.init(trustStore);
+
+ // initialize the ssl context
+ final SSLContext sslContext = SSLContext.getInstance(protocol);
+ sslContext.init(keyManagerFactory.getKeyManagers(), trustManagerFactory.getTrustManagers(), new SecureRandom());
+ if (ClientAuth.REQUIRED == clientAuth) {
+ sslContext.getDefaultSSLParameters().setNeedClientAuth(true);
+ } else if (ClientAuth.WANT == clientAuth) {
+ sslContext.getDefaultSSLParameters().setWantClientAuth(true);
+ } else {
+ sslContext.getDefaultSSLParameters().setWantClientAuth(false);
+ }
+
+ return sslContext;
+
+ }
+
+ /**
+ * Creates a SSLContext instance using the given information.
+ *
+ * @param keystore the full path to the keystore
+ * @param keystorePasswd the keystore password
+ * @param keystoreType the type of keystore (e.g., PKCS12, JKS)
+ * @param protocol the protocol to use for the SSL connection
+ *
+ * @return a SSLContext instance
+ * @throws java.security.KeyStoreException if any issues accessing the keystore
+ * @throws java.io.IOException for any problems loading the keystores
+ * @throws java.security.NoSuchAlgorithmException if an algorithm is found to be used but is unknown
+ * @throws java.security.cert.CertificateException if there is an issue with the certificate
+ * @throws java.security.UnrecoverableKeyException if the key is insufficient
+ * @throws java.security.KeyManagementException if unable to manage the key
+ */
+ public static SSLContext createSslContext(
+ final String keystore, final char[] keystorePasswd, final String keystoreType, final String protocol)
+ throws KeyStoreException, IOException, NoSuchAlgorithmException, CertificateException,
+ UnrecoverableKeyException, KeyManagementException {
+
+ // prepare the keystore
+ final KeyStore keyStore = KeyStore.getInstance(keystoreType);
+ try (final InputStream keyStoreStream = new FileInputStream(keystore)) {
+ keyStore.load(keyStoreStream, keystorePasswd);
+ }
+ final KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
+ keyManagerFactory.init(keyStore, keystorePasswd);
+
+ // initialize the ssl context
+ final SSLContext ctx = SSLContext.getInstance(protocol);
+ ctx.init(keyManagerFactory.getKeyManagers(), new TrustManager[0], new SecureRandom());
+
+ return ctx;
+
+ }
+
+ /**
+ * Creates a SSLContext instance using the given information.
+ *
+ * @param truststore the full path to the truststore
+ * @param truststorePasswd the truststore password
+ * @param truststoreType the type of truststore (e.g., PKCS12, JKS)
+ * @param protocol the protocol to use for the SSL connection
+ *
+ * @return a SSLContext instance
+ * @throws java.security.KeyStoreException if any issues accessing the keystore
+ * @throws java.io.IOException for any problems loading the keystores
+ * @throws java.security.NoSuchAlgorithmException if an algorithm is found to be used but is unknown
+ * @throws java.security.cert.CertificateException if there is an issue with the certificate
+ * @throws java.security.UnrecoverableKeyException if the key is insufficient
+ * @throws java.security.KeyManagementException if unable to manage the key
+ */
+ public static SSLContext createTrustSslContext(
+ final String truststore, final char[] truststorePasswd, final String truststoreType, final String protocol)
+ throws KeyStoreException, IOException, NoSuchAlgorithmException, CertificateException,
+ UnrecoverableKeyException, KeyManagementException {
+
+ // prepare the truststore
+ final KeyStore trustStore = KeyStore.getInstance(truststoreType);
+ try (final InputStream trustStoreStream = new FileInputStream(truststore)) {
+ trustStore.load(trustStoreStream, truststorePasswd);
+ }
+ final TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
+ trustManagerFactory.init(trustStore);
+
+ // initialize the ssl context
+ final SSLContext ctx = SSLContext.getInstance(protocol);
+ ctx.init(new KeyManager[0], trustManagerFactory.getTrustManagers(), new SecureRandom());
+
+ return ctx;
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/pom.xml b/nifi-commons/nifi-site-to-site-client/pom.xml
new file mode 100644
index 0000000..01b3aec
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/pom.xml
@@ -0,0 +1,54 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-commons</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-site-to-site-client</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-utils</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-mapper-asl</artifactId>
+ <version>1.9.13</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-client-dto</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/AbstractCommunicationsSession.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/AbstractCommunicationsSession.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/AbstractCommunicationsSession.java
new file mode 100644
index 0000000..dacfd64
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/AbstractCommunicationsSession.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote;
+
+import org.apache.nifi.remote.protocol.CommunicationsSession;
+
+public abstract class AbstractCommunicationsSession implements CommunicationsSession {
+
+ private String userDn;
+
+ private volatile String uri;
+
+ public AbstractCommunicationsSession(final String uri) {
+ this.uri = uri;
+ }
+
+ @Override
+ public String toString() {
+ return uri;
+ }
+
+ @Override
+ public void setUri(final String uri) {
+ this.uri = uri;
+ }
+
+ @Override
+ public String getUri() {
+ return uri;
+ }
+
+ @Override
+ public String getUserDn() {
+ return userDn;
+ }
+
+ @Override
+ public void setUserDn(final String dn) {
+ this.userDn = dn;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/Communicant.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/Communicant.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/Communicant.java
new file mode 100644
index 0000000..17b990e
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/Communicant.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote;
+
+/**
+ * Represents the remote entity that the client is communicating with
+ */
+public interface Communicant {
+
+ /**
+ * @return the NiFi site-to-site URL for the remote NiFi instance
+ */
+ String getUrl();
+
+ /**
+ * @return The Host of the remote NiFi instance
+ */
+ String getHost();
+
+ /**
+ * @return The Port that the remote NiFi instance is listening on for
+ * site-to-site communications
+ */
+ int getPort();
+
+ /**
+ * @return The distinguished name that the remote NiFi instance has provided
+ * in its certificate if using secure communications, or <code>null</code>
+ * if the Distinguished Name is unknown
+ */
+ String getDistinguishedName();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/Peer.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/Peer.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/Peer.java
new file mode 100644
index 0000000..5cb37b0
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/Peer.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.nifi.remote.protocol.CommunicationsSession;
+
+public class Peer implements Communicant {
+
+ private final PeerDescription description;
+ private final CommunicationsSession commsSession;
+ private final String url;
+ private final String clusterUrl;
+ private final String host;
+ private final int port;
+
+ private final Map<String, Long> penaltyExpirationMap = new HashMap<>();
+ private boolean closed = false;
+
+ public Peer(final PeerDescription description, final CommunicationsSession commsSession, final String peerUrl, final String clusterUrl) {
+ this.description = description;
+ this.commsSession = commsSession;
+ this.url = peerUrl;
+ this.clusterUrl = clusterUrl;
+
+ try {
+ final URI uri = new URI(peerUrl);
+ this.port = uri.getPort();
+ this.host = uri.getHost();
+ } catch (final Exception e) {
+ throw new IllegalArgumentException("Invalid URL: " + peerUrl);
+ }
+ }
+
+ public PeerDescription getDescription() {
+ return description;
+ }
+
+ @Override
+ public String getUrl() {
+ return url;
+ }
+
+ public String getClusterUrl() {
+ return clusterUrl;
+ }
+
+ public CommunicationsSession getCommunicationsSession() {
+ return commsSession;
+ }
+
+ public void close() throws IOException {
+ this.closed = true;
+
+ // Consume the InputStream so that it doesn't linger on the Peer's outgoing socket buffer
+ try {
+ commsSession.getInput().consume();
+ } finally {
+ commsSession.close();
+ }
+ }
+
+ /**
+ * Penalizes this peer for the given destination only for the provided
+ * number of milliseconds
+ *
+ * @param destinationId id of destination
+ * @param millis period of time to penalize peer
+ */
+ public void penalize(final String destinationId, final long millis) {
+ final Long currentPenalty = penaltyExpirationMap.get(destinationId);
+ final long proposedPenalty = System.currentTimeMillis() + millis;
+ if (currentPenalty == null || proposedPenalty > currentPenalty) {
+ penaltyExpirationMap.put(destinationId, proposedPenalty);
+ }
+ }
+
+ public boolean isPenalized(final String destinationId) {
+ final Long currentPenalty = penaltyExpirationMap.get(destinationId);
+ return (currentPenalty != null && currentPenalty > System.currentTimeMillis());
+ }
+
+ public boolean isClosed() {
+ return closed;
+ }
+
+ @Override
+ public String getHost() {
+ return host;
+ }
+
+ @Override
+ public int hashCode() {
+ return 8320 + url.hashCode();
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (obj == null) {
+ return false;
+ }
+ if (obj == this) {
+ return true;
+ }
+ if (!(obj instanceof Peer)) {
+ return false;
+ }
+
+ final Peer other = (Peer) obj;
+ return this.url.equals(other.url);
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder();
+ sb.append("Peer[url=").append(url);
+ if (closed) {
+ sb.append(",CLOSED");
+ }
+ sb.append("]");
+ return sb.toString();
+ }
+
+ @Override
+ public int getPort() {
+ return port;
+ }
+
+ @Override
+ public String getDistinguishedName() {
+ return commsSession.getUserDn();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/PeerDescription.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/PeerDescription.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/PeerDescription.java
new file mode 100644
index 0000000..6fc90e4
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/PeerDescription.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote;
+
+public class PeerDescription {
+
+ private final String hostname;
+ private final int port;
+ private final boolean secure;
+
+ public PeerDescription(final String hostname, final int port, final boolean secure) {
+ this.hostname = hostname;
+ this.port = port;
+ this.secure = secure;
+ }
+
+ public String getHostname() {
+ return hostname;
+ }
+
+ public int getPort() {
+ return port;
+ }
+
+ public boolean isSecure() {
+ return secure;
+ }
+
+ @Override
+ public String toString() {
+ return "PeerDescription[hostname=" + hostname + ", port=" + port + ", secure=" + secure + "]";
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((hostname == null) ? 0 : hostname.hashCode());
+ result = prime * result + port;
+ return result;
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+
+ final PeerDescription other = (PeerDescription) obj;
+ if (hostname == null) {
+ if (other.hostname != null) {
+ return false;
+ }
+ } else if (!hostname.equals(other.hostname)) {
+ return false;
+ }
+
+ return port == other.port;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/PeerStatus.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/PeerStatus.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/PeerStatus.java
new file mode 100644
index 0000000..6c8a4ec
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/PeerStatus.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote;
+
+public class PeerStatus {
+
+ private final PeerDescription description;
+ private final int numFlowFiles;
+
+ public PeerStatus(final PeerDescription description, final int numFlowFiles) {
+ this.description = description;
+ this.numFlowFiles = numFlowFiles;
+ }
+
+ public PeerDescription getPeerDescription() {
+ return description;
+ }
+
+ public int getFlowFileCount() {
+ return numFlowFiles;
+ }
+
+ @Override
+ public String toString() {
+ return "PeerStatus[hostname=" + description.getHostname() + ",port=" + description.getPort()
+ + ",secure=" + description.isSecure() + ",flowFileCount=" + numFlowFiles + "]";
+ }
+
+ @Override
+ public int hashCode() {
+ return 9824372 + description.getHostname().hashCode() + description.getPort() * 41;
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (obj == null) {
+ return false;
+ }
+
+ if (!(obj instanceof PeerStatus)) {
+ return false;
+ }
+
+ final PeerStatus other = (PeerStatus) obj;
+ return description.equals(other.getPeerDescription());
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/RemoteResourceInitiator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/RemoteResourceInitiator.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/RemoteResourceInitiator.java
new file mode 100644
index 0000000..582916e
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/RemoteResourceInitiator.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.nifi.remote.exception.HandshakeException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class RemoteResourceInitiator {
+
+ public static final int RESOURCE_OK = 20;
+ public static final int DIFFERENT_RESOURCE_VERSION = 21;
+ public static final int ABORT = 255;
+
+ private static final Logger logger = LoggerFactory.getLogger(RemoteResourceInitiator.class);
+
+ public static VersionedRemoteResource initiateResourceNegotiation(final VersionedRemoteResource resource, final DataInputStream dis, final DataOutputStream dos)
+ throws IOException, HandshakeException {
+ // Write the classname of the RemoteStreamCodec, followed by its version
+ logger.debug("Negotiating resource; proposal is {}", resource);
+ dos.writeUTF(resource.getResourceName());
+ final VersionNegotiator negotiator = resource.getVersionNegotiator();
+ dos.writeInt(negotiator.getVersion());
+ dos.flush();
+
+ // wait for response from server.
+ logger.debug("Receiving response from remote instance");
+ final int statusCode = dis.read();
+ switch (statusCode) {
+ case RESOURCE_OK: // server accepted our proposal of codec name/version
+ logger.debug("Response was RESOURCE_OK");
+ return resource;
+ case DIFFERENT_RESOURCE_VERSION: // server accepted our proposal of codec name but not the version
+ logger.debug("Response was DIFFERENT_RESOURCE_VERSION");
+ // Get server's preferred version
+ final int newVersion = dis.readInt();
+
+ // Determine our new preferred version that is no greater than the server's preferred version.
+ final Integer newPreference = negotiator.getPreferredVersion(newVersion);
+ // If we could not agree with server on a version, fail now.
+ if (newPreference == null) {
+ throw new HandshakeException("Could not agree on version for " + resource);
+ }
+
+ negotiator.setVersion(newPreference);
+
+ // Attempt negotiation of resource based on our new preferred version.
+ return initiateResourceNegotiation(resource, dis, dos);
+ case ABORT:
+ logger.debug("Response was ABORT");
+ throw new HandshakeException("Remote destination aborted connection with message: " + dis.readUTF());
+ default:
+ logger.debug("Response was {}; unable to negotiate codec", statusCode);
+ return null; // Unable to negotiate codec
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/Transaction.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/Transaction.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/Transaction.java
new file mode 100644
index 0000000..bfa5c82
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/Transaction.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.nifi.remote.protocol.DataPacket;
+
+/**
+ * <p>
+ * Provides a transaction for performing site-to-site data transfers.
+ * </p>
+ *
+ * <p>
+ * A Transaction is created by calling the
+ * {@link org.apache.nifi.remote.client.SiteToSiteClient#createTransaction(TransferDirection) createTransaction(TransferDirection)}
+ * method of a
+ * {@link org.apache.nifi.remote.client.SiteToSiteClient SiteToSiteClient}. The
+ * resulting Transaction can be used to either send or receive data but not
+ * both. A new Transaction must be created in order perform the other operation.
+ * </p>
+ *
+ * <p>
+ * The general flow of execute of a Transaction is as follows:
+ * <ol>
+ * <li>Create the transaction as described above.</li>
+ * <li>Send data via the {@link #send(DataPacket)} method or receive data via
+ * the {@link #receive()} method. This method will be called 1 or more times. In
+ * the case of receive, this method should be called until the method returns
+ * {@code null}, signifying that the remote instance is finished sending data.
+ * <b>Note:</b> <code>receive()</code> should not be called a second time
+ * without first fully consuming the stream from the previous Packet that was
+ * received.</li>
+ * <li>Confirm the transaction via the {@link #confirm()} method.</li>
+ * <li>Either complete the transaction via the {@link #complete(boolean)} method
+ * or cancel the transaction via the {@link #cancel()} method.</li>
+ * </ol>
+ * </p>
+ *
+ * <p>
+ * It is important that the Transaction be terminated in order to free the
+ * resources held by the Transaction. If a Transaction is not terminated, its
+ * resources will not be freed and if the Transaction holds connections from a
+ * connection pool, the connections in that pool will eventually become
+ * exhausted. A Transaction is terminated by calling one of the following
+ * methods:
+ * <ul>
+ * <li>{@link #complete(boolean)}</li>
+ * <li>{@link #cancel()}</li>
+ * <li>{@link #error()}</li>
+ * </ul>
+ * </p>
+ *
+ * <p>
+ * If at any point an IOException is thrown from one of the methods of the
+ * Transaction, that Transaction is automatically closed via a call to
+ * {@link #error()}.
+ * </p>
+ *
+ * <p>
+ * The Transaction class should not be assumed to be thread-safe.
+ * </p>
+ */
+public interface Transaction {
+
+ /**
+ * Sends information to the remote NiFi instance.
+ *
+ * @param dataPacket the data packet to send
+ * @throws IOException if unable to send
+ */
+ void send(DataPacket dataPacket) throws IOException;
+
+ /**
+ * Sends the given byte array as the content of a {@link DataPacket} along
+ * with the provided attributes
+ *
+ * @param content to send
+ * @param attributes of the content
+ * @throws IOException if unable to send
+ */
+ void send(byte[] content, Map<String, String> attributes) throws IOException;
+
+ /**
+ * Retrieves information from the remote NiFi instance, if any is available.
+ * If no data is available, will return {@code null}. It is important to
+ * consume all data from the remote NiFi instance before attempting to call
+ * {@link #confirm()}. This is because the sender is always responsible for
+ * determining when the Transaction has finished. This is done in order to
+ * prevent the need for a round-trip network request to receive data for
+ * each data packet.
+ *
+ * @return the DataPacket received, or {@code null} if there is no more data
+ * to receive.
+ * @throws IOException if unable to receive
+ */
+ DataPacket receive() throws IOException;
+
+ /**
+ * <p>
+ * Confirms the data that was sent or received by comparing CRC32's of the
+ * data sent and the data received.
+ * </p>
+ *
+ * <p>
+ * Even if the protocol being used to send the data is reliable and
+ * guarantees ordering of packets (such as TCP), it is still required that
+ * we confirm the transaction before completing the transaction. This is
+ * done as "safety net" or a defensive programming technique. Mistakes
+ * happen, and this mechanism helps to ensure that if a bug exists somewhere
+ * along the line that we do not end up sending or receiving corrupt data.
+ * If the CRC32 of the sender and the CRC32 of the receiver do not match, an
+ * IOException will be thrown and both the sender and receiver will cancel
+ * the transaction automatically.
+ * </p>
+ *
+ * <p>
+ * If the {@link TransferDirection} of this Transaction is RECEIVE, this
+ * method will throw an Exception unless all data from the remote instance
+ * has been consumed (i.e., a call to {@link #receive()} returns
+ * {@code null}).
+ * </p>
+ *
+ * <p>
+ * If the {@link TransferDirection} of this Transaction is SEND, calling
+ * this method dictates that no more data will be sent in this transaction.
+ * I.e., there will be no more calls to {@link #send(DataPacket)}.
+ * </p>
+ *
+ * @throws IOException if unable to confirm transaction
+ */
+ void confirm() throws IOException;
+
+ /**
+ * <p>
+ * Completes the transaction and indicates to both the sender and receiver
+ * that the data transfer was successful.
+ * </p>
+ *
+ * @throws IOException if unable to complete
+ *
+ * @return a TransactionCompletion that contains details about the
+ * Transaction
+ */
+ TransactionCompletion complete() throws IOException;
+
+ /**
+ * <p>
+ * Cancels this transaction, indicating to the sender that the data has not
+ * been successfully received so that the sender can retry or handle however
+ * is appropriate.
+ * </p>
+ *
+ * @param explanation an explanation to tell the other party why the
+ * transaction was canceled.
+ * @throws IOException if unable to cancel
+ */
+ void cancel(final String explanation) throws IOException;
+
+ /**
+ * <p>
+ * Sets the TransactionState of the Transaction to
+ * {@link TransactionState#ERROR}, and closes the Transaction. The
+ * underlying connection should not be returned to a connection pool in this
+ * case.
+ * </p>
+ */
+ void error();
+
+ /**
+ * @return the current state of the Transaction.
+ * @throws IOException ioe
+ */
+ TransactionState getState() throws IOException;
+
+ /**
+ * @return a Communicant that represents the other side of this Transaction
+ * (i.e., the remote NiFi instance)
+ */
+ Communicant getCommunicant();
+
+ public enum TransactionState {
+
+ /**
+ * Transaction has been started but no data has been sent or received.
+ */
+ TRANSACTION_STARTED,
+ /**
+ * Transaction has been started and data has been sent or received.
+ */
+ DATA_EXCHANGED,
+ /**
+ * Data that has been transferred has been confirmed via its CRC.
+ * Transaction is ready to be completed.
+ */
+ TRANSACTION_CONFIRMED,
+ /**
+ * Transaction has been successfully completed.
+ */
+ TRANSACTION_COMPLETED,
+ /**
+ * The Transaction has been canceled.
+ */
+ TRANSACTION_CANCELED,
+ /**
+ * The Transaction ended in an error.
+ */
+ ERROR;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/TransactionCompletion.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/TransactionCompletion.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/TransactionCompletion.java
new file mode 100644
index 0000000..1587e87
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/TransactionCompletion.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote;
+
+import java.io.InputStream;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.nifi.remote.protocol.DataPacket;
+
+/**
+ * A TransactionCompletion provides information about a {@link Transaction} that
+ * has completed successfully.
+ */
+public interface TransactionCompletion {
+
+ /**
+ * When a sending to a NiFi instance, the server may accept the content sent
+ * to it but indicate that its queues are full and that the client should
+ * backoff sending data for a bit.
+ *
+ * @return <code>true</code> if the server did in fact request that,
+ * <code>false</code> otherwise
+ */
+ boolean isBackoff();
+
+ /**
+ * @return the number of Data Packets that were sent to or received from the
+ * remote NiFi instance in the Transaction
+ */
+ int getDataPacketsTransferred();
+
+ /**
+ * @return the number of bytes of DataPacket content that were sent to or
+ * received from the remote NiFI instance in the Transaction. Note that this
+ * is different than the number of bytes actually transferred between the
+ * client and server, as it does not take into account the attributes or
+ * protocol-specific information that is exchanged but rather takes into
+ * account only the data in the {@link InputStream} of the
+ * {@link DataPacket}
+ */
+ long getBytesTransferred();
+
+ /**
+ * @param timeUnit unit of time for which to report the duration
+ * @return the amount of time that the Transaction took, from the time that
+ * the Transaction was created to the time that the Transaction was
+ * completed
+ */
+ long getDuration(TimeUnit timeUnit);
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/TransferDirection.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/TransferDirection.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/TransferDirection.java
new file mode 100644
index 0000000..979ad9c
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/TransferDirection.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote;
+
+/**
+ * An enumeration for specifying the direction in which data should be
+ * transferred between a client and a remote NiFi instance.
+ */
+public enum TransferDirection {
+
+ /**
+ * The client is to send data to the remote instance.
+ */
+ SEND,
+ /**
+ * The client is to receive data from the remote instance.
+ */
+ RECEIVE;
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/VersionedRemoteResource.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/VersionedRemoteResource.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/VersionedRemoteResource.java
new file mode 100644
index 0000000..bfccd98
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/VersionedRemoteResource.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote;
+
+public interface VersionedRemoteResource {
+
+ VersionNegotiator getVersionNegotiator();
+
+ String getResourceName();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/client/KeystoreType.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/client/KeystoreType.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/client/KeystoreType.java
new file mode 100644
index 0000000..63c3d63
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/client/KeystoreType.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.client;
+
+import java.io.Serializable;
+
+public enum KeystoreType implements Serializable {
+ PKCS12,
+ JKS;
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/client/SiteToSiteClient.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/client/SiteToSiteClient.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/client/SiteToSiteClient.java
new file mode 100644
index 0000000..78237b9
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/client/SiteToSiteClient.java
@@ -0,0 +1,771 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.client;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Serializable;
+import java.security.KeyStore;
+import java.security.SecureRandom;
+import java.util.concurrent.TimeUnit;
+
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManagerFactory;
+
+import org.apache.nifi.events.EventReporter;
+import org.apache.nifi.remote.Transaction;
+import org.apache.nifi.remote.TransferDirection;
+import org.apache.nifi.remote.client.socket.SocketClient;
+import org.apache.nifi.remote.exception.HandshakeException;
+import org.apache.nifi.remote.exception.PortNotRunningException;
+import org.apache.nifi.remote.exception.ProtocolException;
+import org.apache.nifi.remote.exception.UnknownPortException;
+import org.apache.nifi.remote.protocol.DataPacket;
+
+/**
+ * <p>
+ * The SiteToSiteClient provides a mechanism for sending data to a remote
+ * instance of NiFi (or NiFi cluster) and retrieving data from a remote instance
+ * of NiFi (or NiFi cluster).
+ * </p>
+ *
+ * <p>
+ * When configuring the client via the {@link SiteToSiteClient.Builder}, the
+ * Builder must be provided the URL of the remote NiFi instance. If the URL
+ * points to a standalone instance of NiFi, all interaction will take place with
+ * that instance of NiFi. However, if the URL points to the NiFi Cluster Manager
+ * of a cluster, the client will automatically handle load balancing the
+ * transactions across the different nodes in the cluster.
+ * </p>
+ *
+ * <p>
+ * The SiteToSiteClient provides a {@link Transaction} through which all
+ * interaction with the remote instance takes place. After data has been
+ * exchanged or it is determined that no data is available, the Transaction can
+ * then be canceled (via the {@link Transaction#cancel(String)} method) or can
+ * be completed (via the {@link Transaction#complete(boolean)} method).
+ * </p>
+ *
+ * <p>
+ * An instance of SiteToSiteClient can be obtained by constructing a new
+ * instance of the {@link SiteToSiteClient.Builder} class, calling the
+ * appropriate methods to configured the client as desired, and then calling the
+ * {@link SiteToSiteClient.Builder#build() build()} method.
+ * </p>
+ *
+ * <p>
+ * The SiteToSiteClient itself is immutable once constructed and is thread-safe.
+ * Many threads can share access to the same client. However, the
+ * {@link Transaction} that is created by the client is not thread safe and
+ * should not be shared among threads.
+ * </p>
+ */
+public interface SiteToSiteClient extends Closeable {
+
+ /**
+ * <p>
+ * Creates a new Transaction that can be used to either send data to a
+ * remote NiFi instance or receive data from a remote NiFi instance,
+ * depending on the value passed for the {@code direction} argument.
+ * </p>
+ *
+ * <p>
+ * <b>Note:</b> If all of the nodes are penalized (See
+ * {@link Builder#nodePenalizationPeriod(long, TimeUnit)}), then this method
+ * will return <code>null</code>.
+ * </p>
+ *
+ * @param direction specifies which direction the data should be
+ * transferred. A value of {@link TransferDirection#SEND} indicates that
+ * this Transaction will send data to the remote instance; a value of
+ * {@link TransferDirection#RECEIVE} indicates that this Transaction will be
+ * used to receive data from the remote instance.
+ *
+ * @return a Transaction to use for sending or receiving data, or
+ * <code>null</code> if all nodes are penalized.
+ * @throws org.apache.nifi.remote.exception.HandshakeException he
+ * @throws org.apache.nifi.remote.exception.PortNotRunningException pnre
+ * @throws IOException ioe
+ * @throws org.apache.nifi.remote.exception.UnknownPortException upe
+ */
+ Transaction createTransaction(TransferDirection direction) throws HandshakeException, PortNotRunningException, ProtocolException, UnknownPortException, IOException;
+
+ /**
+ * <p>
+ * In order to determine whether the server is configured for secure
+ * communications, the client may have to query the server's RESTful
+ * interface. Doing so could result in an IOException.
+ * </p>
+ *
+ * @return {@code true} if site-to-site communications with the remote
+ * instance are secure, {@code false} if site-to-site communications with
+ * the remote instance are not secure. Whether or not communications are
+ * secure depends on the server, not the client
+ * @throws IOException if unable to query the remote instance's RESTful
+ * interface or if the remote instance is not configured to allow
+ * site-to-site communications
+ */
+ boolean isSecure() throws IOException;
+
+ /**
+ *
+ * @return the configuration object that was built by the Builder
+ */
+ SiteToSiteClientConfig getConfig();
+
+ /**
+ * <p>
+ * The Builder is the mechanism by which all configuration is passed to the
+ * SiteToSiteClient. Once constructed, the SiteToSiteClient cannot be
+ * reconfigured (i.e., it is immutable). If a change in configuration should
+ * be desired, the client should be {@link Closeable#close() closed} and a
+ * new client created.
+ * </p>
+ */
+ public static class Builder implements Serializable {
+
+ private static final long serialVersionUID = -4954962284343090219L;
+
+ private String url;
+ private long timeoutNanos = TimeUnit.SECONDS.toNanos(30);
+ private long penalizationNanos = TimeUnit.SECONDS.toNanos(3);
+ private long idleExpirationNanos = TimeUnit.SECONDS.toNanos(30L);
+ private SSLContext sslContext;
+ private String keystoreFilename;
+ private String keystorePass;
+ private KeystoreType keystoreType;
+ private String truststoreFilename;
+ private String truststorePass;
+ private KeystoreType truststoreType;
+ private EventReporter eventReporter;
+ private File peerPersistenceFile;
+ private boolean useCompression;
+ private String portName;
+ private String portIdentifier;
+ private int batchCount;
+ private long batchSize;
+ private long batchNanos;
+
+ /**
+ * Populates the builder with values from the provided config
+ *
+ * @param config to start with
+ * @return the builder
+ */
+ public Builder fromConfig(final SiteToSiteClientConfig config) {
+ this.url = config.getUrl();
+ this.timeoutNanos = config.getTimeout(TimeUnit.NANOSECONDS);
+ this.penalizationNanos = config.getPenalizationPeriod(TimeUnit.NANOSECONDS);
+ this.idleExpirationNanos = config.getIdleConnectionExpiration(TimeUnit.NANOSECONDS);
+ this.sslContext = config.getSslContext();
+ this.keystoreFilename = config.getKeystoreFilename();
+ this.keystorePass = config.getKeystorePassword();
+ this.keystoreType = config.getKeystoreType();
+ this.truststoreFilename = config.getTruststoreFilename();
+ this.truststorePass = config.getTruststorePassword();
+ this.truststoreType = config.getTruststoreType();
+ this.eventReporter = config.getEventReporter();
+ this.peerPersistenceFile = config.getPeerPersistenceFile();
+ this.useCompression = config.isUseCompression();
+ this.portName = config.getPortName();
+ this.portIdentifier = config.getPortIdentifier();
+ this.batchCount = config.getPreferredBatchCount();
+ this.batchSize = config.getPreferredBatchSize();
+ this.batchNanos = config.getPreferredBatchDuration(TimeUnit.NANOSECONDS);
+
+ return this;
+ }
+
+ /**
+ * Specifies the URL of the remote NiFi instance. If this URL points to
+ * the Cluster Manager of a NiFi cluster, data transfer to and from
+ * nodes will be automatically load balanced across the different nodes.
+ *
+ * @param url url of remote instance
+ * @return the builder
+ */
+ public Builder url(final String url) {
+ this.url = url;
+ return this;
+ }
+
+ /**
+ * Specifies the communications timeouts to use when interacting with
+ * the remote instances. The default value is 30 seconds.
+ *
+ * @param timeout to use when interacting with remote instances
+ * @param unit unit of time over which to interpret the given timeout
+ * @return the builder
+ */
+ public Builder timeout(final long timeout, final TimeUnit unit) {
+ this.timeoutNanos = unit.toNanos(timeout);
+ return this;
+ }
+
+ /**
+ * Specifies the amount of time that a connection can remain idle in the
+ * connection pool before it is "expired" and shutdown. The default
+ * value is 30 seconds.
+ *
+ * @param timeout to use when interacting with remote instances
+ * @param unit unit of time over which to interpret the given timeout
+ * @return the builder
+ */
+ public Builder idleExpiration(final long timeout, final TimeUnit unit) {
+ this.idleExpirationNanos = unit.toNanos(timeout);
+ return this;
+ }
+
+ /**
+ * If there is a problem communicating with a node (i.e., any node in
+ * the remote NiFi cluster or the remote instance of NiFi if it is
+ * standalone), specifies how long the client should wait before
+ * attempting to communicate with that node again. While a particular
+ * node is penalized, all other nodes in the remote cluster (if any)
+ * will still be available for communication. The default value is 3
+ * seconds.
+ *
+ * @param period time to wait between communication attempts
+ * @param unit over which to evaluate the given period
+ * @return the builder
+ */
+ public Builder nodePenalizationPeriod(final long period, final TimeUnit unit) {
+ this.penalizationNanos = unit.toNanos(period);
+ return this;
+ }
+
+ /**
+ * Specifies the SSL Context to use when communicating with the remote
+ * NiFi instance(s). If not specified, communications will not be
+ * secure. The remote instance of NiFi always determines whether or not
+ * Site-to-Site communications are secure (i.e., the client will always
+ * use secure or non-secure communications, depending on what the server
+ * dictates). <b>Note:</b> The SSLContext provided by this method will be
+ * ignored if using a Serializable Configuration (see {@link #buildSerializableConfig()}).
+ * If a Serializable Configuration is required and communications are to be
+ * secure, the {@link #keystoreFilename(String)}, {@link #keystorePass(String)},
+ * {@link #keystoreType}, {@link #truststoreFilename}, {@link #truststorePass(String)},
+ * and {@link #truststoreType(KeystoreType)} methods must be used instead.
+ *
+ * @param sslContext the context
+ * @return the builder
+ */
+ public Builder sslContext(final SSLContext sslContext) {
+ this.sslContext = sslContext;
+ return this;
+ }
+
+ /**
+ * @return the filename to use for the Keystore in order to communicate securely
+ * with the remote instance of NiFi
+ */
+ public String getKeystoreFilename() {
+ return keystoreFilename;
+ }
+
+ /**
+ * Sets the filename to use for the Keystore in order to communicate securely
+ * with the remote instance of NiFi
+ *
+ * @param keystoreFilename the filename to use for the Keystore in order to communicate securely
+ * with the remote instance of NiFi
+ * @return the builder
+ */
+ public Builder keystoreFilename(final String keystoreFilename) {
+ this.keystoreFilename = keystoreFilename;
+ return this;
+ }
+
+ /**
+ * @return the password to use for the Keystore in order to communicate securely
+ * with the remote instance of NiFi
+ */
+ public String getKeystorePass() {
+ return keystorePass;
+ }
+
+ /**
+ * Sets the password to use for the Keystore in order to communicate securely
+ * with the remote instance of NiFi
+ *
+ * @param keystorePass the password to use for the Keystore in order to communicate securely
+ * with the remote instance of NiFi
+ * @return the builder
+ */
+ public Builder keystorePass(final String keystorePass) {
+ this.keystorePass = keystorePass;
+ return this;
+ }
+
+ /**
+ * @return the type of Keystore to use in order to communicate securely
+ * with the remote instance of NiFi
+ */
+ public KeystoreType getKeystoreType() {
+ return keystoreType;
+ }
+
+ /**
+ * Sets the type of the Keystore to use in order to communicate securely
+ * with the remote instance of NiFi
+ *
+ * @param keystoreType the type of the Keystore to use in order to communicate securely
+ * with the remote instance of NiFi
+ * @return the builder
+ */
+ public Builder keystoreType(final KeystoreType keystoreType) {
+ this.keystoreType = keystoreType;
+ return this;
+ }
+
+ /**
+ * @return the filename to use for the Truststore in order to communicate securely
+ * with the remote instance of NiFi
+ */
+ public String getTruststoreFilename() {
+ return truststoreFilename;
+ }
+
+ /**
+ * Sets the filename to use for the Truststore in order to communicate securely
+ * with the remote instance of NiFi
+ *
+ * @param truststoreFilename the filename to use for the Truststore in order to communicate securely
+ * with the remote instance of NiFi
+ * @return the builder
+ */
+ public Builder truststoreFilename(final String truststoreFilename) {
+ this.truststoreFilename = truststoreFilename;
+ return this;
+ }
+
+ /**
+ * @return the password to use for the Truststore in order to communicate securely
+ * with the remote instance of NiFi
+ */
+ public String getTruststorePass() {
+ return truststorePass;
+ }
+
+ /**
+ * Sets the password to use for the Truststore in order to communicate securely
+ * with the remote instance of NiFi
+ *
+ * @param truststorePass the filename to use for the Truststore in order to communicate securely
+ * with the remote instance of NiFi
+ */
+ public Builder truststorePass(final String truststorePass) {
+ this.truststorePass = truststorePass;
+ return this;
+ }
+
+ /**
+ * @return the type of the Truststore to use in order to communicate securely
+ * with the remote instance of NiFi
+ */
+ public KeystoreType getTruststoreType() {
+ return truststoreType;
+ }
+
+ /**
+ * Sets the password type of the Truststore to use in order to communicate securely
+ * with the remote instance of NiFi
+ *
+ * @param truststoreType the type of the Truststore to use in order to communicate securely
+ * with the remote instance of NiFi
+ * @return the builder
+ */
+ public Builder truststoreType(final KeystoreType truststoreType) {
+ this.truststoreType = truststoreType;
+ return this;
+ }
+
+ /**
+ * Provides an EventReporter that can be used by the client in order to
+ * report any events that could be of interest when communicating with
+ * the remote instance. The EventReporter provided must be threadsafe.
+ *
+ * @param eventReporter reporter
+ * @return the builder
+ */
+ public Builder eventReporter(final EventReporter eventReporter) {
+ this.eventReporter = eventReporter;
+ return this;
+ }
+
+ /**
+ * Specifies a file that the client can write to in order to persist the
+ * list of nodes in the remote cluster and recover the list of nodes
+ * upon restart. This allows the client to function if the remote
+ * Cluster Manager is unavailable, even after a restart of the client
+ * software. If not specified, the list of nodes will not be persisted
+ * and a failure of the Cluster Manager will result in not being able to
+ * communicate with the remote instance if a new client is created.
+ *
+ * @param peerPersistenceFile file
+ * @return the builder
+ */
+ public Builder peerPersistenceFile(final File peerPersistenceFile) {
+ this.peerPersistenceFile = peerPersistenceFile;
+ return this;
+ }
+
+ /**
+ * Specifies whether or not data should be compressed before being
+ * transferred to or from the remote instance.
+ *
+ * @param compress true if should compress
+ * @return the builder
+ */
+ public Builder useCompression(final boolean compress) {
+ this.useCompression = compress;
+ return this;
+ }
+
+ /**
+ * Specifies the name of the port to communicate with. Either the port
+ * name or the port identifier must be specified.
+ *
+ * @param portName name of port
+ * @return the builder
+ */
+ public Builder portName(final String portName) {
+ this.portName = portName;
+ return this;
+ }
+
+ /**
+ * Specifies the unique identifier of the port to communicate with. If
+ * it is known, this is preferred over providing the port name, as the
+ * port name may change.
+ *
+ * @param portIdentifier identifier of port
+ * @return the builder
+ */
+ public Builder portIdentifier(final String portIdentifier) {
+ this.portIdentifier = portIdentifier;
+ return this;
+ }
+
+ /**
+ * When pulling data from a NiFi instance, the sender chooses how large
+ * a Transaction is. However, the client has the ability to request a
+ * particular batch size/duration. This method specifies the preferred
+ * number of {@link DataPacket}s to include in a Transaction.
+ *
+ * @param count client preferred batch size
+ * @return the builder
+ */
+ public Builder requestBatchCount(final int count) {
+ this.batchCount = count;
+ return this;
+ }
+
+ /**
+ * When pulling data from a NiFi instance, the sender chooses how large
+ * a Transaction is. However, the client has the ability to request a
+ * particular batch size/duration. This method specifies the preferred
+ * number of bytes to include in a Transaction.
+ *
+ * @param bytes client preferred batch size
+ * @return the builder
+ */
+ public Builder requestBatchSize(final long bytes) {
+ this.batchSize = bytes;
+ return this;
+ }
+
+ /**
+ * When pulling data from a NiFi instance, the sender chooses how large
+ * a Transaction is. However, the client has the ability to request a
+ * particular batch size/duration. This method specifies the preferred
+ * amount of time that a Transaction should span.
+ *
+ * @param value client preferred batch duration
+ * @param unit client preferred batch duration unit
+ * @return the builder
+ */
+ public Builder requestBatchDuration(final long value, final TimeUnit unit) {
+ this.batchNanos = unit.toNanos(value);
+ return this;
+ }
+
+ /**
+ * @return a {@link SiteToSiteClientConfig} for the configured values
+ * but does not create a SiteToSiteClient
+ */
+ public SiteToSiteClientConfig buildConfig() {
+ return new SiteToSiteClientConfig() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public boolean isUseCompression() {
+ return Builder.this.isUseCompression();
+ }
+
+ @Override
+ public String getUrl() {
+ return Builder.this.getUrl();
+ }
+
+ @Override
+ public long getTimeout(final TimeUnit timeUnit) {
+ return Builder.this.getTimeout(timeUnit);
+ }
+
+ @Override
+ public long getIdleConnectionExpiration(final TimeUnit timeUnit) {
+ return Builder.this.getIdleConnectionExpiration(timeUnit);
+ }
+
+ @Override
+ public SSLContext getSslContext() {
+ return Builder.this.getSslContext();
+ }
+
+ @Override
+ public String getPortName() {
+ return Builder.this.getPortName();
+ }
+
+ @Override
+ public String getPortIdentifier() {
+ return Builder.this.getPortIdentifier();
+ }
+
+ @Override
+ public long getPenalizationPeriod(final TimeUnit timeUnit) {
+ return Builder.this.getPenalizationPeriod(timeUnit);
+ }
+
+ @Override
+ public File getPeerPersistenceFile() {
+ return Builder.this.getPeerPersistenceFile();
+ }
+
+ @Override
+ public EventReporter getEventReporter() {
+ return Builder.this.getEventReporter();
+ }
+
+ @Override
+ public long getPreferredBatchDuration(final TimeUnit timeUnit) {
+ return timeUnit.convert(batchNanos, TimeUnit.NANOSECONDS);
+ }
+
+ @Override
+ public long getPreferredBatchSize() {
+ return batchSize;
+ }
+
+ @Override
+ public int getPreferredBatchCount() {
+ return batchCount;
+ }
+
+ @Override
+ public String getKeystoreFilename() {
+ return keystoreFilename;
+ }
+
+ @Override
+ public String getKeystorePassword() {
+ return keystorePass;
+ }
+
+ @Override
+ public KeystoreType getKeystoreType() {
+ return keystoreType;
+ }
+
+ @Override
+ public String getTruststoreFilename() {
+ return truststoreFilename;
+ }
+
+ @Override
+ public String getTruststorePassword() {
+ return truststorePass;
+ }
+
+ @Override
+ public KeystoreType getTruststoreType() {
+ return truststoreType;
+ }
+ };
+ }
+
+ /**
+ * @return a new SiteToSiteClient that can be used to send and receive
+ * data with remote instances of NiFi
+ *
+ * @throws IllegalStateException if either the url is not set or neither
+ * the port name nor port identifier is set.
+ */
+ public SiteToSiteClient build() {
+ if (url == null) {
+ throw new IllegalStateException("Must specify URL to build Site-to-Site client");
+ }
+
+ if (portName == null && portIdentifier == null) {
+ throw new IllegalStateException("Must specify either Port Name or Port Identifier to build Site-to-Site client");
+ }
+
+ return new SocketClient(buildConfig());
+ }
+
+ /**
+ * @return the configured URL for the remote NiFi instance
+ */
+ public String getUrl() {
+ return url;
+ }
+
+ /**
+ * @param timeUnit unit over which to interpret the timeout
+ * @return the communications timeout
+ */
+ public long getTimeout(final TimeUnit timeUnit) {
+ return timeUnit.convert(timeoutNanos, TimeUnit.NANOSECONDS);
+ }
+
+ /**
+ * @param timeUnit unit over which to interpret the time
+ * @return the amount of of time that a connection can remain idle in
+ * the connection pool before being shutdown
+ */
+ public long getIdleConnectionExpiration(final TimeUnit timeUnit) {
+ return timeUnit.convert(idleExpirationNanos, TimeUnit.NANOSECONDS);
+ }
+
+ /**
+ * @param timeUnit unit of reported time
+ * @return the amount of time that a particular node will be ignored
+ * after a communications error with that node occurs
+ */
+ public long getPenalizationPeriod(TimeUnit timeUnit) {
+ return timeUnit.convert(penalizationNanos, TimeUnit.NANOSECONDS);
+ }
+
+ /**
+ * @return the SSL Context that is configured for this builder
+ */
+ public SSLContext getSslContext() {
+ if (sslContext != null) {
+ return sslContext;
+ }
+
+ final KeyManagerFactory keyManagerFactory;
+ if (keystoreFilename != null && keystorePass != null && keystoreType != null) {
+ try {
+ // prepare the keystore
+ final KeyStore keyStore = KeyStore.getInstance(getKeystoreType().name());
+ try (final InputStream keyStoreStream = new FileInputStream(new File(getKeystoreFilename()))) {
+ keyStore.load(keyStoreStream, getKeystorePass().toCharArray());
+ }
+ keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
+ keyManagerFactory.init(keyStore, getKeystorePass().toCharArray());
+ } catch (final Exception e) {
+ throw new RuntimeException("Failed to load Keystore", e);
+ }
+ } else {
+ keyManagerFactory = null;
+ }
+
+ final TrustManagerFactory trustManagerFactory;
+ if (truststoreFilename != null && truststorePass != null && truststoreType != null) {
+ try {
+ // prepare the truststore
+ final KeyStore trustStore = KeyStore.getInstance(getTruststoreType().name());
+ try (final InputStream trustStoreStream = new FileInputStream(new File(getTruststoreFilename()))) {
+ trustStore.load(trustStoreStream, getTruststorePass().toCharArray());
+ }
+ trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
+ trustManagerFactory.init(trustStore);
+ } catch (final Exception e) {
+ throw new RuntimeException("Failed to load Truststore", e);
+ }
+ } else {
+ trustManagerFactory = null;
+ }
+
+ if (keyManagerFactory != null || trustManagerFactory != null) {
+ try {
+ // initialize the ssl context
+ final SSLContext sslContext = SSLContext.getInstance("TLS");
+ sslContext.init(keyManagerFactory.getKeyManagers(), trustManagerFactory.getTrustManagers(), new SecureRandom());
+ sslContext.getDefaultSSLParameters().setNeedClientAuth(true);
+
+ return sslContext;
+ } catch (final Exception e) {
+ throw new RuntimeException("Created keystore and truststore but failed to initialize SSLContext");
+ }
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * @return the EventReporter that is to be used by clients to report
+ * events
+ */
+ public EventReporter getEventReporter() {
+ return eventReporter;
+ }
+
+ /**
+ * @return the file that is to be used for persisting the nodes of a
+ * remote cluster, if any
+ */
+ public File getPeerPersistenceFile() {
+ return peerPersistenceFile;
+ }
+
+ /**
+ * @return a boolean indicating whether or not compression will be used
+ * to transfer data to and from the remote instance
+ */
+ public boolean isUseCompression() {
+ return useCompression;
+ }
+
+ /**
+ * @return the name of the port that the client is to communicate with
+ */
+ public String getPortName() {
+ return portName;
+ }
+
+ /**
+ * @return the identifier of the port that the client is to communicate
+ * with
+ */
+ public String getPortIdentifier() {
+ return portIdentifier;
+ }
+ }
+
+
+ public abstract class SerializableSiteToSiteClientConfig implements SiteToSiteClientConfig, Serializable {
+ private static final long serialVersionUID = 1L;
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/client/SiteToSiteClientConfig.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/client/SiteToSiteClientConfig.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/client/SiteToSiteClientConfig.java
new file mode 100644
index 0000000..50a0d3c
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/client/SiteToSiteClientConfig.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.client;
+
+import java.io.File;
+import java.io.Serializable;
+import java.util.concurrent.TimeUnit;
+
+import javax.net.ssl.SSLContext;
+
+import org.apache.nifi.events.EventReporter;
+import org.apache.nifi.remote.protocol.DataPacket;
+
+public interface SiteToSiteClientConfig extends Serializable {
+
+ /**
+ * @return the configured URL for the remote NiFi instance
+ */
+ String getUrl();
+
+ /**
+ * @param timeUnit unit over which to report the timeout
+ * @return the communications timeout in given unit
+ */
+ long getTimeout(final TimeUnit timeUnit);
+
+ /**
+ * @param timeUnit the unit for which to report the time
+ * @return the amount of time that a connection can remain idle before it is
+ * "expired" and shut down
+ */
+ long getIdleConnectionExpiration(TimeUnit timeUnit);
+
+ /**
+ * @param timeUnit unit over which to report the time
+ * @return the amount of time that a particular node will be ignored after a
+ * communications error with that node occurs
+ */
+ long getPenalizationPeriod(TimeUnit timeUnit);
+
+ /**
+ * @return the SSL Context that is configured for this builder
+ */
+ SSLContext getSslContext();
+
+ /**
+ * @return the filename to use for the keystore, or <code>null</code> if none is configured
+ */
+ String getKeystoreFilename();
+
+ /**
+ * @return the password to use for the keystore, or <code>null</code> if none is configured
+ */
+ String getKeystorePassword();
+
+ /**
+ * @return the Type of the keystore, or <code>null</code> if none is configured
+ */
+ KeystoreType getKeystoreType();
+
+ /**
+ * @return the filename to use for the truststore, or <code>null</code> if none is configured
+ */
+ String getTruststoreFilename();
+
+ /**
+ * @return the password to use for the truststore, or <code>null</code> if none is configured
+ */
+ String getTruststorePassword();
+
+ /**
+ * @return the type of the truststore, or <code>null</code> if none is configured
+ */
+ KeystoreType getTruststoreType();
+
+ /**
+ * @return the file that is to be used for persisting the nodes of a remote
+ * cluster, if any
+ */
+ File getPeerPersistenceFile();
+
+ /**
+ * @return a boolean indicating whether or not compression will be used to
+ * transfer data to and from the remote instance
+ */
+ boolean isUseCompression();
+
+ /**
+ * @return the name of the port that the client is to communicate with
+ */
+ String getPortName();
+
+ /**
+ * @return the identifier of the port that the client is to communicate with
+ */
+ String getPortIdentifier();
+
+ /**
+ * When pulling data from a NiFi instance, the sender chooses how large a
+ * Transaction is. However, the client has the ability to request a
+ * particular batch size/duration.
+ *
+ * @param timeUnit unit of time over which to report the duration
+ * @return the maximum amount of time that we will request a NiFi instance
+ * to send data to us in a Transaction
+ */
+ long getPreferredBatchDuration(TimeUnit timeUnit);
+
+ /**
+ * When pulling data from a NiFi instance, the sender chooses how large a
+ * Transaction is. However, the client has the ability to request a
+ * particular batch size/duration.
+ *
+ * @return returns the maximum number of bytes that we will request a NiFi
+ * instance to send data to us in a Transaction
+ */
+ long getPreferredBatchSize();
+
+ /**
+ * When pulling data from a NiFi instance, the sender chooses how large a
+ * Transaction is. However, the client has the ability to request a
+ * particular batch size/duration.
+ *
+ * @return the maximum number of {@link DataPacket}s that we will request a
+ * NiFi instance to send data to us in a Transaction
+ */
+ int getPreferredBatchCount();
+
+ /**
+ * @return the EventReporter that is to be used by clients to report events
+ */
+ EventReporter getEventReporter();
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/client/socket/EndpointConnection.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/client/socket/EndpointConnection.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/client/socket/EndpointConnection.java
new file mode 100644
index 0000000..1a16b02
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/client/socket/EndpointConnection.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.client.socket;
+
+import org.apache.nifi.remote.Peer;
+import org.apache.nifi.remote.codec.FlowFileCodec;
+import org.apache.nifi.remote.protocol.socket.SocketClientProtocol;
+
+public class EndpointConnection {
+
+ private final Peer peer;
+ private final SocketClientProtocol socketClientProtocol;
+ private final FlowFileCodec codec;
+ private volatile long lastUsed;
+
+ public EndpointConnection(final Peer peer, final SocketClientProtocol socketClientProtocol, final FlowFileCodec codec) {
+ this.peer = peer;
+ this.socketClientProtocol = socketClientProtocol;
+ this.codec = codec;
+ }
+
+ public FlowFileCodec getCodec() {
+ return codec;
+ }
+
+ public SocketClientProtocol getSocketClientProtocol() {
+ return socketClientProtocol;
+ }
+
+ public Peer getPeer() {
+ return peer;
+ }
+
+ public void setLastTimeUsed() {
+ lastUsed = System.currentTimeMillis();
+ }
+
+ public long getLastTimeUsed() {
+ return lastUsed;
+ }
+}
[08/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestPutS3Object.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestPutS3Object.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestPutS3Object.java
new file mode 100644
index 0000000..de7816d
--- /dev/null
+++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestPutS3Object.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.aws.s3;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import com.amazonaws.services.s3.model.StorageClass;
+
+@Ignore("For local testing only - interacts with S3 so the credentials file must be configured and all necessary buckets created")
+public class TestPutS3Object {
+
+ private final String CREDENTIALS_FILE = System.getProperty("user.home") + "/aws-credentials.properties";
+
+ @Test
+ public void testSimplePut() throws IOException {
+ final TestRunner runner = TestRunners.newTestRunner(new PutS3Object());
+ runner.setProperty(PutS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
+ runner.setProperty(PutS3Object.BUCKET, "test-bucket-00000000-0000-0000-0000-123456789012");
+ runner.setProperty(PutS3Object.EXPIRATION_RULE_ID, "Expire Quickly");
+ Assert.assertTrue(runner.setProperty("x-custom-prop", "hello").isValid());
+
+ for (int i = 0; i < 3; i++) {
+ final Map<String, String> attrs = new HashMap<>();
+ attrs.put("filename", String.valueOf(i) + ".txt");
+ runner.enqueue(Paths.get("src/test/resources/hello.txt"), attrs);
+ }
+ runner.run(3);
+
+ runner.assertAllFlowFilesTransferred(PutS3Object.REL_SUCCESS, 3);
+ }
+
+ @Test
+ public void testPutInFolder() throws IOException {
+ final TestRunner runner = TestRunners.newTestRunner(new PutS3Object());
+ runner.setProperty(PutS3Object.BUCKET, "test-bucket-00000000-0000-0000-0000-123456789012");
+ runner.setProperty(PutS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
+ runner.setProperty(PutS3Object.EXPIRATION_RULE_ID, "Expire Quickly");
+ Assert.assertTrue(runner.setProperty("x-custom-prop", "hello").isValid());
+
+ final Map<String, String> attrs = new HashMap<>();
+ attrs.put("filename", "folder/1.txt");
+ runner.enqueue(Paths.get("src/test/resources/hello.txt"), attrs);
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred(PutS3Object.REL_SUCCESS, 1);
+ }
+
+ @Test
+ public void testStorageClass() throws IOException {
+ final TestRunner runner = TestRunners.newTestRunner(new PutS3Object());
+ runner.setProperty(PutS3Object.BUCKET, "test-bucket-00000000-0000-0000-0000-123456789012");
+ runner.setProperty(PutS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
+ runner.setProperty(PutS3Object.STORAGE_CLASS, StorageClass.ReducedRedundancy.name());
+ Assert.assertTrue(runner.setProperty("x-custom-prop", "hello").isValid());
+
+ final Map<String, String> attrs = new HashMap<>();
+ attrs.put("filename", "folder/2.txt");
+ runner.enqueue(Paths.get("src/test/resources/hello.txt"), attrs);
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred(PutS3Object.REL_SUCCESS, 1);
+ }
+
+ @Test
+ public void testPermissions() throws IOException {
+ final TestRunner runner = TestRunners.newTestRunner(new PutS3Object());
+ runner.setProperty(PutS3Object.BUCKET, "test-bucket-00000000-0000-0000-0000-123456789012");
+ runner.setProperty(PutS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
+ runner.setProperty(PutS3Object.FULL_CONTROL_USER_LIST, "28545acd76c35c7e91f8409b95fd1aa0c0914bfa1ac60975d9f48bc3c5e090b5");
+
+ final Map<String, String> attrs = new HashMap<>();
+ attrs.put("filename", "folder/4.txt");
+ runner.enqueue(Paths.get("src/test/resources/hello.txt"), attrs);
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred(PutS3Object.REL_SUCCESS, 1);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/sns/TestPutSNS.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/sns/TestPutSNS.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/sns/TestPutSNS.java
new file mode 100644
index 0000000..1e914c7
--- /dev/null
+++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/sns/TestPutSNS.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.aws.sns;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.Ignore;
+import org.junit.Test;
+
+@Ignore("For local testing only - interacts with S3 so the credentials file must be configured and all necessary buckets created")
+public class TestPutSNS {
+
+ private final String CREDENTIALS_FILE = System.getProperty("user.home") + "/aws-credentials.properties";
+
+ @Test
+ public void testPublish() throws IOException {
+ final TestRunner runner = TestRunners.newTestRunner(new PutSNS());
+ runner.setProperty(PutSNS.CREDENTAILS_FILE, CREDENTIALS_FILE);
+ runner.setProperty(PutSNS.ARN, "arn:aws:sns:us-west-2:100515378163:test-topic-1");
+ assertTrue(runner.setProperty("DynamicProperty", "hello!").isValid());
+
+ final Map<String, String> attrs = new HashMap<>();
+ attrs.put("filename", "1.txt");
+ runner.enqueue(Paths.get("src/test/resources/hello.txt"), attrs);
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred(PutSNS.REL_SUCCESS, 1);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/sqs/TestGetSQS.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/sqs/TestGetSQS.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/sqs/TestGetSQS.java
new file mode 100644
index 0000000..0e70e7b
--- /dev/null
+++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/sqs/TestGetSQS.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.aws.sqs;
+
+import java.util.List;
+
+import org.apache.nifi.processors.aws.sns.PutSNS;
+import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.Ignore;
+import org.junit.Test;
+
+@Ignore("For local testing only - interacts with S3 so the credentials file must be configured and all necessary buckets created")
+public class TestGetSQS {
+
+ private final String CREDENTIALS_FILE = System.getProperty("user.home") + "/aws-credentials.properties";
+
+ @Test
+ public void testSimpleGet() {
+ final TestRunner runner = TestRunners.newTestRunner(new GetSQS());
+ runner.setProperty(PutSNS.CREDENTAILS_FILE, CREDENTIALS_FILE);
+ runner.setProperty(GetSQS.TIMEOUT, "30 secs");
+ runner.setProperty(GetSQS.QUEUE_URL, "https://sqs.us-west-2.amazonaws.com/100515378163/test-queue-000000000");
+
+ runner.run(1);
+
+ final List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(GetSQS.REL_SUCCESS);
+ for (final MockFlowFile mff : flowFiles) {
+ System.out.println(mff.getAttributes());
+ System.out.println(new String(mff.toByteArray()));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/sqs/TestPutSQS.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/sqs/TestPutSQS.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/sqs/TestPutSQS.java
new file mode 100644
index 0000000..1f9851a
--- /dev/null
+++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/sqs/TestPutSQS.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.aws.sqs;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.nifi.processors.aws.sns.PutSNS;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
+
+@Ignore("For local testing only - interacts with S3 so the credentials file must be configured and all necessary buckets created")
+public class TestPutSQS {
+
+ private final String CREDENTIALS_FILE = System.getProperty("user.home") + "/aws-credentials.properties";
+
+ @Test
+ public void testSimplePut() throws IOException {
+ final TestRunner runner = TestRunners.newTestRunner(new PutSQS());
+ runner.setProperty(PutSNS.CREDENTAILS_FILE, CREDENTIALS_FILE);
+ runner.setProperty(PutSQS.TIMEOUT, "30 secs");
+ runner.setProperty(PutSQS.QUEUE_URL, "https://sqs.us-west-2.amazonaws.com/100515378163/test-queue-000000000");
+ Assert.assertTrue(runner.setProperty("x-custom-prop", "hello").isValid());
+
+ final Map<String, String> attrs = new HashMap<>();
+ attrs.put("filename", "1.txt");
+ runner.enqueue(Paths.get("src/test/resources/hello.txt"), attrs);
+ runner.run(1);
+
+ runner.assertAllFlowFilesTransferred(PutSQS.REL_SUCCESS, 1);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/resources/hello.txt
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/resources/hello.txt b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/resources/hello.txt
new file mode 100644
index 0000000..ee13cb7
--- /dev/null
+++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/resources/hello.txt
@@ -0,0 +1 @@
+Hello, World!!
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-aws-bundle/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-aws-bundle/pom.xml b/nifi-nar-bundles/nifi-aws-bundle/pom.xml
new file mode 100644
index 0000000..1c3642c
--- /dev/null
+++ b/nifi-nar-bundles/nifi-aws-bundle/pom.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-nar-bundles</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-aws-bundle</artifactId>
+ <packaging>pom</packaging>
+
+ <modules>
+ <module>nifi-aws-processors</module>
+ <module>nifi-aws-nar</module>
+ </modules>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk</artifactId>
+ <version>1.9.24</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-nar/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-nar/pom.xml b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-nar/pom.xml
new file mode 100644
index 0000000..5180f6d
--- /dev/null
+++ b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-nar/pom.xml
@@ -0,0 +1,132 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-flume-bundle</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>nifi-flume-nar</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ <packaging>nar</packaging>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-flume-processors</artifactId>
+ <!-- The following are inherited from nifi-hadoop-libraries-nar -->
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.thoughtworks.paranamer</groupId>
+ <artifactId>paranamer</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-mapper-asl</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-core-asl</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-collections</groupId>
+ <artifactId>commons-collections</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-hadoop-libraries-nar</artifactId>
+ <type>nar</type>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-nar/src/main/resources/META-INF/LICENSE
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-nar/src/main/resources/META-INF/LICENSE b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-nar/src/main/resources/META-INF/LICENSE
new file mode 100644
index 0000000..c1a3ec4
--- /dev/null
+++ b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-nar/src/main/resources/META-INF/LICENSE
@@ -0,0 +1,319 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+APACHE NIFI SUBCOMPONENTS:
+
+The Apache NiFi project contains subcomponents with separate copyright
+notices and license terms. Your use of the source code for the these
+subcomponents is subject to the terms and conditions of the following
+licenses.
+
+The binary distribution of this product bundles 'SUAsync Library' which is
+available under a 3-Clause BSD License.
+
+ Copyright (c) 2010 StumbleUpon, Inc. All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+ - Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+ - Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+ - Neither the name of the StumbleUpon nor the names of its contributors
+ may be used to endorse or promote products derived from this software
+ without specific prior written permission.
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ POSSIBILITY OF SUCH DAMAGE.
+
+The binary distribution of this product bundles 'Asynchronous HBase Client'
+which is available under a 3-Clause BSD License.
+
+ Copyright (C) 2010-2012 The Async HBase Authors. All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+ - Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+ - Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+ - Neither the name of the StumbleUpon nor the names of its contributors
+ may be used to endorse or promote products derived from this software
+ without specific prior written permission.
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ POSSIBILITY OF SUCH DAMAGE.
+
+The binary distribution of this product bundles 'JOpt Simple' which is
+available under the MIT license.
+
+ The MIT License
+
+ Copyright (c) 2004-2011 Paul R. Holser, Jr.
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+The binary distribution of this product bundles 'Scala Library' under a BSD
+style license.
+
+ Copyright (c) 2002-2015 EPFL
+ Copyright (c) 2011-2015 Typesafe, Inc.
+
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without modification,
+ are permitted provided that the following conditions are met:
+
+ Redistributions of source code must retain the above copyright notice, this list of
+ conditions and the following disclaimer.
+
+ Redistributions in binary form must reproduce the above copyright notice, this list of
+ conditions and the following disclaimer in the documentation and/or other materials
+ provided with the distribution.
+
+ Neither the name of the EPFL nor the names of its contributors may be used to endorse
+ or promote products derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” AND ANY EXPRESS
+ OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+ AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-nar/src/main/resources/META-INF/NOTICE
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-nar/src/main/resources/META-INF/NOTICE b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-nar/src/main/resources/META-INF/NOTICE
new file mode 100644
index 0000000..b651143
--- /dev/null
+++ b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-nar/src/main/resources/META-INF/NOTICE
@@ -0,0 +1,192 @@
+nifi-flume-nar
+Copyright 2015 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+******************
+Apache Software License v2
+******************
+
+The following binary components are provided under the Apache Software License v2
+
+ (ASLv2) Apache Avro
+ The following NOTICE information applies:
+ Apache Avro
+ Copyright 2009-2012 The Apache Software Foundation
+
+ (ASLv2) Apache Commons JEXL
+ The following NOTICE information applies:
+ Apache Commons JEXL
+ Copyright 2001-2011 The Apache Software Foundation
+
+ (ASLv2) Apache Commons Lang
+ The following NOTICE information applies:
+ Apache Commons Lang
+ Copyright 2001-2015 The Apache Software Foundation
+
+ This product includes software from the Spring Framework,
+ under the Apache License 2.0 (see: StringUtils.containsWhitespace())
+
+ (ASLv2) Apache Flume
+ The following NOTICE information applies:
+ Apache Flume
+ Copyright 2011-2015 Apache Software Foundation
+
+ asynchbase is BSD-licensed software (https://github.com/OpenTSDB/asynchbase)
+
+ async is BSD-licensed software (https://github.com/stumbleupon/async)
+
+ jopt-simple is MIT licensed software (http://pholser.github.io/jopt-simple/license.html)
+
+ scala-library is BSD-like licensed software (http://www.scala-lang.org/license.html)
+
+ (ASLv2) Xalan
+ This product includes software developed by
+ The Apache Software Foundation (http://www.apache.org/).
+
+ Portions of this software was originally based on the following:
+
+ - software copyright (c) 1999-2002, Lotus Development Corporation., http://www.lotus.com.
+ - software copyright (c) 2001-2002, Sun Microsystems., http://www.sun.com.
+ - software copyright (c) 2003, IBM Corporation., http://www.ibm.com.
+ - voluntary contributions made by Ovidiu Predescu (ovidiu@cup.hp.com) on behalf of the
+ Apache Software Foundation and was originally developed at Hewlett Packard Company.
+
+ (ASLv2) Apache Xerces Java
+ Copyright 1999-2007 The Apache Software Foundation
+
+ This product includes software developed at
+ The Apache Software Foundation (http://www.apache.org/).
+
+ Portions of this software were originally based on the following:
+ - software copyright (c) 1999, IBM Corporation., http://www.ibm.com.
+ - software copyright (c) 1999, Sun Microsystems., http://www.sun.com.
+ - voluntary contributions made by Paul Eng on behalf of the
+ Apache Software Foundation that were originally developed at iClick, Inc.,
+ software copyright (c) 1999.
+
+ (ASLv2) Apache XML Commons XML APIs
+ Copyright 2006 The Apache Software Foundation.
+
+ This product includes software developed at
+ The Apache Software Foundation (http://www.apache.org/).
+
+ Portions of this software were originally based on the following:
+ - software copyright (c) 1999, IBM Corporation., http://www.ibm.com.
+ - software copyright (c) 1999, Sun Microsystems., http://www.sun.com.
+ - software copyright (c) 2000 World Wide Web Consortium, http://www.w3.org
+
+ (ASLv2) IRClib
+ The following NOTICE information applies:
+ IRClib -- A Java Internet Relay Chat library --
+ Copyright (C) 2002 - 2006 Christoph Schwering <sc...@gmail.com>
+
+ (ASLv2) Jackson JSON processor
+ The following NOTICE information applies:
+ # Jackson JSON processor
+
+ Jackson is a high-performance, Free/Open Source JSON processing library.
+ It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has
+ been in development since 2007.
+ It is currently developed by a community of developers, as well as supported
+ commercially by FasterXML.com.
+
+ ## Licensing
+
+ Jackson core and extension components may licensed under different licenses.
+ To find the details that apply to this artifact see the accompanying LICENSE file.
+ For more information, including possible other licensing options, contact
+ FasterXML.com (http://fasterxml.com).
+
+ ## Credits
+
+ A list of contributors may be found from CREDITS file, which is included
+ in some artifacts (usually source distributions); but is always available
+ from the source code management (SCM) system project uses.
+
+ (ASLv2) Joda-Time
+ The following NOTICE information applies:
+ =============================================================================
+ = NOTICE file corresponding to section 4d of the Apache License Version 2.0 =
+ =============================================================================
+ This product includes software developed by
+ Joda.org (http://www.joda.org/).
+
+ (ASLv2) Apache Kafka
+ The following NOTICE information applies:
+ Apache Kafka
+ Copyright 2012 The Apache Software Foundation.
+
+ (ASLv2) Kite SDK
+ The following NOTICE information applies:
+ This product includes software developed by Cloudera, Inc.
+ (http://www.cloudera.com/).
+
+ This product includes software developed at
+ The Apache Software Foundation (http://www.apache.org/).
+
+ This product includes software developed by
+ Saxonica (http://www.saxonica.com/).
+
+ (ASLv2) Apache Thrift
+ The following NOTICE information applies:
+ Apache Thrift
+ Copyright 2006-2010 The Apache Software Foundation.
+
+ (ASLv2) Yammer Metrics
+ The following NOTICE information applies:
+ Metrics
+ Copyright 2010-2012 Coda Hale and Yammer, Inc.
+
+ This product includes software developed by Coda Hale and Yammer, Inc.
+
+ This product includes code derived from the JSR-166 project (ThreadLocalRandom), which was released
+ with the following comments:
+
+ Written by Doug Lea with assistance from members of JCP JSR-166
+ Expert Group and released to the public domain, as explained at
+ http://creativecommons.org/publicdomain/zero/1.0/
+
+ (ASLv2) Apache MINA
+ The following NOTICE information applies:
+ Apache MINA Core
+ Copyright 2004-2011 Apache MINA Project
+
+ (ASLv2) The Netty Project
+ The following NOTICE information applies:
+ The Netty Project
+ Copyright 2011 The Netty Project
+
+ (ASLv2) opencsv (net.sf.opencsv:opencsv:2.3)
+
+ (ASLv2) Parquet MR
+ The following NOTICE information applies:
+ Parquet MR
+ Copyright 2012 Twitter, Inc.
+
+ This project includes code from https://github.com/lemire/JavaFastPFOR
+ parquet-column/src/main/java/parquet/column/values/bitpacking/LemireBitPacking.java
+ Apache License Version 2.0 http://www.apache.org/licenses/.
+ (c) Daniel Lemire, http://lemire.me/en/
+
+ (ASLv2) Jetty
+ The following NOTICE information applies:
+ Jetty Web Container
+ Copyright 1995-2015 Mort Bay Consulting Pty Ltd.
+
+ (ASLv2) Twitter4J
+ The following NOTICE information applies:
+ Copyright 2007 Yusuke Yamamoto
+
+ Twitter4J includes software from JSON.org to parse JSON response from the Twitter API. You can see the license term at http://www.JSON.org/license.html
+
+ (ASLv2) Apache Velocity
+ The following NOTICE information applies:
+ Apache Velocity
+ Copyright (C) 2000-2007 The Apache Software Foundation
+
+ (ASLv2) ZkClient
+ The following NOTICE information applies:
+ ZkClient
+ Copyright 2009 Stefan Groschupf
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/pom.xml b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/pom.xml
new file mode 100644
index 0000000..e793255
--- /dev/null
+++ b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/pom.xml
@@ -0,0 +1,178 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-flume-bundle</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>nifi-flume-processors</artifactId>
+ <packaging>jar</packaging>
+
+ <properties>
+ <flume.version>1.6.0</flume.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-utils</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-processor-utils</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-flowfile-packager</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-ng-sdk</artifactId>
+ <version>${flume.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-ng-core</artifactId>
+ <version>${flume.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <!-- Flume Sources -->
+
+ <dependency>
+ <groupId>org.apache.flume.flume-ng-sources</groupId>
+ <artifactId>flume-jms-source</artifactId>
+ <version>${flume.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flume.flume-ng-sources</groupId>
+ <artifactId>flume-kafka-source</artifactId>
+ <version>${flume.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flume.flume-ng-sources</groupId>
+ <artifactId>flume-scribe-source</artifactId>
+ <version>${flume.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flume.flume-ng-sources</groupId>
+ <artifactId>flume-twitter-source</artifactId>
+ <version>${flume.version}</version>
+ </dependency>
+
+ <!-- Flume Sinks -->
+ <dependency>
+ <groupId>org.apache.flume.flume-ng-sinks</groupId>
+ <artifactId>flume-dataset-sink</artifactId>
+ <version>${flume.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flume.flume-ng-sinks</groupId>
+ <artifactId>flume-hdfs-sink</artifactId>
+ <version>${flume.version}</version>
+ </dependency>
+
+ <!-- HDFS sink dependencies -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flume.flume-ng-sinks</groupId>
+ <artifactId>flume-hive-sink</artifactId>
+ <version>${flume.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flume.flume-ng-sinks</groupId>
+ <artifactId>flume-irc-sink</artifactId>
+ <version>${flume.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flume.flume-ng-sinks</groupId>
+ <artifactId>flume-ng-elasticsearch-sink</artifactId>
+ <version>${flume.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flume.flume-ng-sinks</groupId>
+ <artifactId>flume-ng-hbase-sink</artifactId>
+ <version>${flume.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flume.flume-ng-sinks</groupId>
+ <artifactId>flume-ng-kafka-sink</artifactId>
+ <version>${flume.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flume.flume-ng-sinks</groupId>
+ <artifactId>flume-ng-morphline-solr-sink</artifactId>
+ <version>${flume.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-mock</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <!-- test data -->
+ <exclude>src/test/resources/testdata/*</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <systemPropertyVariables>
+ <java.io.tmpdir>${project.build.directory}</java.io.tmpdir>
+ </systemPropertyVariables>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/AbstractFlumeProcessor.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/AbstractFlumeProcessor.java b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/AbstractFlumeProcessor.java
new file mode 100644
index 0000000..9b75047
--- /dev/null
+++ b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/AbstractFlumeProcessor.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.flume;
+
+import com.google.common.collect.Maps;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.StringReader;
+import java.util.Map;
+import java.util.Properties;
+import org.apache.flume.Context;
+import org.apache.flume.Event;
+import org.apache.flume.SinkFactory;
+import org.apache.flume.SourceFactory;
+import org.apache.flume.sink.DefaultSinkFactory;
+import org.apache.flume.source.DefaultSourceFactory;
+
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.components.Validator;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.AbstractSessionFactoryProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.ProcessSessionFactory;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.io.OutputStreamCallback;
+import org.apache.nifi.processors.flume.util.FlowFileEvent;
+
+/**
+ * This is a base class that is helpful when building processors interacting with Flume.
+ */
+public abstract class AbstractFlumeProcessor extends AbstractSessionFactoryProcessor {
+
+ protected static final SourceFactory SOURCE_FACTORY = new DefaultSourceFactory();
+ protected static final SinkFactory SINK_FACTORY = new DefaultSinkFactory();
+
+ protected static Event flowFileToEvent(FlowFile flowFile, ProcessSession session) {
+ return new FlowFileEvent(flowFile, session);
+ }
+
+ protected static void transferEvent(final Event event, ProcessSession session,
+ Relationship relationship) {
+ FlowFile flowFile = session.create();
+ flowFile = session.putAllAttributes(flowFile, event.getHeaders());
+
+ flowFile = session.write(flowFile, new OutputStreamCallback() {
+ @Override
+ public void process(final OutputStream out) throws IOException {
+ out.write(event.getBody());
+ }
+ });
+
+ session.getProvenanceReporter()
+ .create(flowFile);
+ session.transfer(flowFile, relationship);
+ }
+
+ protected static Validator createSourceValidator() {
+ return new Validator() {
+ @Override
+ public ValidationResult validate(final String subject, final String value, final ValidationContext context) {
+ String reason = null;
+ try {
+ ExecuteFlumeSource.SOURCE_FACTORY.create("NiFi Source", value);
+ } catch (Exception ex) {
+ reason = ex.getLocalizedMessage();
+ reason = Character.toLowerCase(reason.charAt(0)) + reason.substring(1);
+ }
+ return new ValidationResult.Builder().subject(subject)
+ .input(value)
+ .explanation(reason)
+ .valid(reason == null)
+ .build();
+ }
+ };
+ }
+
+ protected static Validator createSinkValidator() {
+ return new Validator() {
+ @Override
+ public ValidationResult validate(final String subject, final String value, final ValidationContext context) {
+ String reason = null;
+ try {
+ ExecuteFlumeSink.SINK_FACTORY.create("NiFi Sink", value);
+ } catch (Exception ex) {
+ reason = ex.getLocalizedMessage();
+ reason = Character.toLowerCase(reason.charAt(0)) + reason.substring(1);
+ }
+ return new ValidationResult.Builder().subject(subject)
+ .input(value)
+ .explanation(reason)
+ .valid(reason == null)
+ .build();
+ }
+ };
+ }
+
+ protected static Context getFlumeContext(String flumeConfig, String prefix) {
+ Properties flumeProperties = new Properties();
+ if (flumeConfig != null) {
+ try {
+ flumeProperties.load(new StringReader(flumeConfig));
+ } catch (IOException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+ Map<String, String> parameters = Maps.newHashMap();
+ for (String property : flumeProperties.stringPropertyNames()) {
+ parameters.put(property, flumeProperties.getProperty(property));
+ }
+ return new Context(new Context(parameters).getSubProperties(prefix));
+ }
+
+ protected static Context getFlumeSourceContext(String flumeConfig,
+ String agentName, String sourceName) {
+ return getFlumeContext(flumeConfig, agentName + ".sources." + sourceName + ".");
+ }
+
+ protected static Context getFlumeSinkContext(String flumeConfig,
+ String agentName, String sinkName) {
+ return getFlumeContext(flumeConfig, agentName + ".sinks." + sinkName + ".");
+ }
+
+ /*
+ * Borrowed from AbstractProcessor. The FlumeSourceProcessor needs to implement this directly
+ * to handle event driven sources, but it's marked final in AbstractProcessor.
+ */
+ @Override
+ public void onTrigger(final ProcessContext context, final ProcessSessionFactory sessionFactory) throws ProcessException {
+ final ProcessSession session = sessionFactory.createSession();
+ try {
+ onTrigger(context, session);
+ session.commit();
+ } catch (final Throwable t) {
+ getLogger()
+ .error("{} failed to process due to {}; rolling back session", new Object[]{this, t});
+ session.rollback(true);
+ throw t;
+ }
+ }
+
+ public abstract void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException;
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/ExecuteFlumeSink.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/ExecuteFlumeSink.java b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/ExecuteFlumeSink.java
new file mode 100644
index 0000000..57e0278
--- /dev/null
+++ b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/ExecuteFlumeSink.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.flume;
+
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import java.util.List;
+import java.util.Set;
+import org.apache.flume.EventDeliveryException;
+import org.apache.flume.Sink;
+import org.apache.flume.conf.Configurables;
+import org.apache.nifi.annotation.behavior.TriggerSerially;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.annotation.lifecycle.OnStopped;
+
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.Validator;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.SchedulingContext;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+
+/**
+ * This processor runs a Flume sink
+ */
+@Tags({"flume", "hadoop", "put", "sink"})
+@CapabilityDescription("Execute a Flume sink. Each input FlowFile is converted into a Flume Event for processing by the sink.")
+@TriggerSerially
+public class ExecuteFlumeSink extends AbstractFlumeProcessor {
+
+ public static final PropertyDescriptor SINK_TYPE = new PropertyDescriptor.Builder()
+ .name("Sink Type")
+ .description("The component type name for the sink. For some sinks, this is a short, symbolic name (e.g. hdfs)."
+ + " For others, it's the fully-qualified name of the Sink class. See the Flume User Guide for details.")
+ .required(true)
+ .addValidator(createSinkValidator())
+ .build();
+ public static final PropertyDescriptor AGENT_NAME = new PropertyDescriptor.Builder()
+ .name("Agent Name")
+ .description("The name of the agent used in the Flume sink configuration")
+ .required(true)
+ .defaultValue("tier1")
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+ public static final PropertyDescriptor SOURCE_NAME = new PropertyDescriptor.Builder()
+ .name("Sink Name")
+ .description("The name of the sink used in the Flume sink configuration")
+ .required(true)
+ .defaultValue("sink-1")
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+ public static final PropertyDescriptor FLUME_CONFIG = new PropertyDescriptor.Builder()
+ .name("Flume Configuration")
+ .description("The Flume configuration for the sink copied from the flume.properties file")
+ .required(true)
+ .defaultValue("")
+ .addValidator(Validator.VALID)
+ .build();
+
+ public static final Relationship SUCCESS = new Relationship.Builder().name("success").build();
+ public static final Relationship FAILURE = new Relationship.Builder().name("failure").build();
+
+ private List<PropertyDescriptor> descriptors;
+ private Set<Relationship> relationships;
+
+ private volatile Sink sink;
+ private volatile NifiSinkSessionChannel channel;
+
+ @Override
+ protected void init(final ProcessorInitializationContext context) {
+ this.descriptors = ImmutableList.of(SINK_TYPE, AGENT_NAME, SOURCE_NAME, FLUME_CONFIG);
+ this.relationships = ImmutableSet.of(SUCCESS, FAILURE);
+ }
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return descriptors;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return relationships;
+ }
+
+ @OnScheduled
+ public void onScheduled(final SchedulingContext context) {
+ try {
+ channel = new NifiSinkSessionChannel(SUCCESS, FAILURE);
+ channel.start();
+
+ sink = SINK_FACTORY.create(context.getProperty(SOURCE_NAME).getValue(),
+ context.getProperty(SINK_TYPE).getValue());
+ sink.setChannel(channel);
+
+ String flumeConfig = context.getProperty(FLUME_CONFIG).getValue();
+ String agentName = context.getProperty(AGENT_NAME).getValue();
+ String sinkName = context.getProperty(SOURCE_NAME).getValue();
+ Configurables.configure(sink,
+ getFlumeSinkContext(flumeConfig, agentName, sinkName));
+
+ sink.start();
+ } catch (Throwable th) {
+ getLogger().error("Error creating sink", th);
+ throw Throwables.propagate(th);
+ }
+ }
+
+ @OnStopped
+ public void stopped() {
+ sink.stop();
+ channel.stop();
+ }
+
+ @Override
+ public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
+
+ channel.setSession(session);
+ try {
+ sink.process();
+ } catch (EventDeliveryException ex) {
+ throw new ProcessException("Flume event delivery failed", ex);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/ExecuteFlumeSource.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/ExecuteFlumeSource.java b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/ExecuteFlumeSource.java
new file mode 100644
index 0000000..70f82cc
--- /dev/null
+++ b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/ExecuteFlumeSource.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.flume;
+
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.flume.EventDeliveryException;
+import org.apache.flume.EventDrivenSource;
+import org.apache.flume.PollableSource;
+import org.apache.flume.Source;
+import org.apache.flume.channel.ChannelProcessor;
+import org.apache.flume.conf.Configurables;
+import org.apache.flume.source.EventDrivenSourceRunner;
+import org.apache.nifi.annotation.behavior.TriggerSerially;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.annotation.lifecycle.OnStopped;
+
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.Validator;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.ProcessSessionFactory;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.SchedulingContext;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+
+/**
+ * This processor runs a Flume source
+ */
+@Tags({"flume", "hadoop", "get", "source"})
+@CapabilityDescription("Execute a Flume source. Each Flume Event is sent to the success relationship as a FlowFile")
+@TriggerSerially
+public class ExecuteFlumeSource extends AbstractFlumeProcessor {
+
+ public static final PropertyDescriptor SOURCE_TYPE = new PropertyDescriptor.Builder()
+ .name("Source Type")
+ .description("The component type name for the source. For some sources, this is a short, symbolic name"
+ + " (e.g. spooldir). For others, it's the fully-qualified name of the Source class. See the Flume User Guide for details.")
+ .required(true)
+ .addValidator(createSourceValidator())
+ .build();
+ public static final PropertyDescriptor AGENT_NAME = new PropertyDescriptor.Builder()
+ .name("Agent Name")
+ .description("The name of the agent used in the Flume source configuration")
+ .required(true)
+ .defaultValue("tier1")
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+ public static final PropertyDescriptor SOURCE_NAME = new PropertyDescriptor.Builder()
+ .name("Source Name")
+ .description("The name of the source used in the Flume source configuration")
+ .required(true)
+ .defaultValue("src-1")
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+ public static final PropertyDescriptor FLUME_CONFIG = new PropertyDescriptor.Builder()
+ .name("Flume Configuration")
+ .description("The Flume configuration for the source copied from the flume.properties file")
+ .required(true)
+ .defaultValue("")
+ .addValidator(Validator.VALID)
+ .build();
+
+ public static final Relationship SUCCESS = new Relationship.Builder().name("success")
+ .build();
+
+ private List<PropertyDescriptor> descriptors;
+ private Set<Relationship> relationships;
+
+ private volatile Source source;
+
+ private final NifiSessionChannel pollableSourceChannel = new NifiSessionChannel(SUCCESS);
+ private final AtomicReference<ProcessSessionFactory> sessionFactoryRef = new AtomicReference<>(null);
+ private final AtomicReference<EventDrivenSourceRunner> runnerRef = new AtomicReference<>(null);
+ private final AtomicReference<NifiSessionFactoryChannel> eventDrivenSourceChannelRef = new AtomicReference<>(null);
+
+ @Override
+ protected void init(final ProcessorInitializationContext context) {
+ this.descriptors = ImmutableList.of(SOURCE_TYPE, AGENT_NAME, SOURCE_NAME, FLUME_CONFIG);
+ this.relationships = ImmutableSet.of(SUCCESS);
+ }
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return descriptors;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return relationships;
+ }
+
+ @OnScheduled
+ public void onScheduled(final SchedulingContext context) {
+ try {
+ source = SOURCE_FACTORY.create(
+ context.getProperty(SOURCE_NAME).getValue(),
+ context.getProperty(SOURCE_TYPE).getValue());
+
+ String flumeConfig = context.getProperty(FLUME_CONFIG).getValue();
+ String agentName = context.getProperty(AGENT_NAME).getValue();
+ String sourceName = context.getProperty(SOURCE_NAME).getValue();
+ Configurables.configure(source,
+ getFlumeSourceContext(flumeConfig, agentName, sourceName));
+
+ if (source instanceof PollableSource) {
+ source.setChannelProcessor(new ChannelProcessor(
+ new NifiChannelSelector(pollableSourceChannel)));
+ source.start();
+ }
+ } catch (Throwable th) {
+ getLogger().error("Error creating source", th);
+ throw Throwables.propagate(th);
+ }
+ }
+
+ @OnStopped
+ public void stopped() {
+ if (source instanceof PollableSource) {
+ source.stop();
+ } else {
+ EventDrivenSourceRunner runner = runnerRef.get();
+ if (runner != null) {
+ runner.stop();
+ runnerRef.compareAndSet(runner, null);
+ }
+
+ NifiSessionFactoryChannel eventDrivenSourceChannel = eventDrivenSourceChannelRef.get();
+ if (eventDrivenSourceChannel != null) {
+ eventDrivenSourceChannel.stop();
+ eventDrivenSourceChannelRef.compareAndSet(eventDrivenSourceChannel, null);
+ }
+ }
+ sessionFactoryRef.set(null);
+ }
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSessionFactory sessionFactory) throws ProcessException {
+ if (source instanceof PollableSource) {
+ super.onTrigger(context, sessionFactory);
+ } else if (source instanceof EventDrivenSource) {
+ ProcessSessionFactory old = sessionFactoryRef.getAndSet(sessionFactory);
+ if (old != sessionFactory) {
+ if (runnerRef.get() != null) {
+ stopped();
+ }
+
+ runnerRef.set(new EventDrivenSourceRunner());
+ eventDrivenSourceChannelRef.set(new NifiSessionFactoryChannel(sessionFactoryRef.get(), SUCCESS));
+ eventDrivenSourceChannelRef.get().start();
+ source.setChannelProcessor(new ChannelProcessor(
+ new NifiChannelSelector(eventDrivenSourceChannelRef.get())));
+ runnerRef.get().setSource(source);
+ runnerRef.get().start();
+ }
+ }
+ }
+
+ @Override
+ public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
+ if (source instanceof PollableSource) {
+ PollableSource pollableSource = (PollableSource) source;
+ try {
+ pollableSourceChannel.setSession(session);
+ pollableSource.process();
+ } catch (EventDeliveryException ex) {
+ throw new ProcessException("Error processing pollable source", ex);
+ }
+ } else {
+ throw new ProcessException("Invalid source type: " + source.getClass().getSimpleName());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/NifiChannelSelector.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/NifiChannelSelector.java b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/NifiChannelSelector.java
new file mode 100644
index 0000000..2b0ba77
--- /dev/null
+++ b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/NifiChannelSelector.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.flume;
+
+import com.google.common.collect.ImmutableList;
+import java.util.List;
+import org.apache.flume.Channel;
+import org.apache.flume.ChannelSelector;
+import org.apache.flume.Context;
+import org.apache.flume.Event;
+
+
+public class NifiChannelSelector implements ChannelSelector {
+ private String name;
+ private final List<Channel> requiredChannels;
+ private final List<Channel> optionalChannels;
+
+ public NifiChannelSelector(Channel channel) {
+ requiredChannels = ImmutableList.of(channel);
+ optionalChannels = ImmutableList.of();
+ }
+
+ @Override
+ public List<Channel> getRequiredChannels(Event event) {
+ return requiredChannels;
+ }
+
+ @Override
+ public List<Channel> getOptionalChannels(Event event) {
+ return optionalChannels;
+ }
+
+ @Override
+ public List<Channel> getAllChannels() {
+ return requiredChannels;
+ }
+
+ @Override
+ public void setChannels(List<Channel> channels) {
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public void configure(Context context) {
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/NifiSessionChannel.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/NifiSessionChannel.java b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/NifiSessionChannel.java
new file mode 100644
index 0000000..4c111af
--- /dev/null
+++ b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/NifiSessionChannel.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.flume;
+
+import org.apache.flume.Context;
+import org.apache.flume.channel.BasicChannelSemantics;
+import org.apache.flume.channel.BasicTransactionSemantics;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+
+public class NifiSessionChannel extends BasicChannelSemantics {
+
+ private ProcessSession session;
+ private final Relationship relationship;
+
+ public NifiSessionChannel(Relationship relationship) {
+ this.relationship = relationship;
+ }
+
+ public void setSession(ProcessSession session) {
+ this.session = session;
+ }
+
+ @Override
+ protected BasicTransactionSemantics createTransaction() {
+ return new NifiTransaction(session, relationship);
+ }
+
+ @Override
+ public void configure(Context context) {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/NifiSessionFactoryChannel.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/NifiSessionFactoryChannel.java b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/NifiSessionFactoryChannel.java
new file mode 100644
index 0000000..eb31a66
--- /dev/null
+++ b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/NifiSessionFactoryChannel.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.flume;
+
+import org.apache.flume.ChannelFullException;
+import org.apache.flume.Context;
+import org.apache.flume.channel.BasicChannelSemantics;
+import org.apache.flume.channel.BasicTransactionSemantics;
+import org.apache.flume.lifecycle.LifecycleState;
+import org.apache.nifi.processor.ProcessSessionFactory;
+import org.apache.nifi.processor.Relationship;
+
+public class NifiSessionFactoryChannel extends BasicChannelSemantics {
+
+ private final ProcessSessionFactory sessionFactory;
+ private final Relationship relationship;
+
+ public NifiSessionFactoryChannel(ProcessSessionFactory sessionFactory, Relationship relationship) {
+ this.sessionFactory = sessionFactory;
+ this.relationship = relationship;
+ }
+
+ @Override
+ protected BasicTransactionSemantics createTransaction() {
+ LifecycleState lifecycleState = getLifecycleState();
+ if (lifecycleState == LifecycleState.STOP) {
+ throw new ChannelFullException("Can't write to a stopped channel");
+ }
+ return new NifiTransaction(sessionFactory.createSession(), relationship);
+ }
+
+ @Override
+ public void configure(Context context) {
+ }
+
+}
[32/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/AbstractNumericComparison.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/AbstractNumericComparison.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/AbstractNumericComparison.java
new file mode 100644
index 0000000..d74e9cf
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/AbstractNumericComparison.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.query.evaluator.comparison;
+
+import java.util.regex.Pattern;
+
+import org.apache.nifi.hl7.query.evaluator.Evaluator;
+
+public abstract class AbstractNumericComparison extends AbstractComparisonEvaluator {
+
+ private static final Pattern NUMERIC_PATTERN = Pattern.compile("\\d+(\\.\\d+)?");
+
+ public AbstractNumericComparison(final Evaluator<?> lhs, final Evaluator<?> rhs) {
+ super(lhs, rhs);
+ }
+
+ @Override
+ protected final boolean compare(final Object lhs, final Object rhs) {
+ final Double lhsDouble = toDouble(lhs);
+ if (lhsDouble == null) {
+ return false;
+ }
+
+ final Double rhsDouble = toDouble(rhs);
+ if (rhsDouble == null) {
+ return false;
+ }
+
+ return compareNumbers(lhsDouble, rhsDouble);
+ }
+
+ private Double toDouble(final Object value) {
+ if (value == null) {
+ return null;
+ }
+
+ if (value instanceof Double) {
+ return (Double) value;
+ }
+ if (value instanceof Number) {
+ return ((Number) value).doubleValue();
+ }
+
+ if (value instanceof String) {
+ if (NUMERIC_PATTERN.matcher((String) value).matches()) {
+ return Double.parseDouble((String) value);
+ }
+ }
+
+ return null;
+ }
+
+ protected abstract boolean compareNumbers(final Double lhs, final Double rhs);
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/EqualsEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/EqualsEvaluator.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/EqualsEvaluator.java
new file mode 100644
index 0000000..fa6c57b
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/EqualsEvaluator.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.query.evaluator.comparison;
+
+import org.apache.nifi.hl7.query.evaluator.Evaluator;
+
+public class EqualsEvaluator extends AbstractComparisonEvaluator {
+
+ public EqualsEvaluator(final Evaluator<?> lhs, final Evaluator<?> rhs) {
+ super(lhs, rhs);
+ }
+
+ @Override
+ protected boolean compare(final Object lhs, final Object rhs) {
+ return lhs != null && rhs != null && ((lhs == rhs) || (lhs.equals(rhs)) || lhs.toString().equals(rhs.toString()));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/GreaterThanEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/GreaterThanEvaluator.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/GreaterThanEvaluator.java
new file mode 100644
index 0000000..cfef7b5
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/GreaterThanEvaluator.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.query.evaluator.comparison;
+
+import org.apache.nifi.hl7.query.evaluator.Evaluator;
+
+public class GreaterThanEvaluator extends AbstractNumericComparison {
+
+ public GreaterThanEvaluator(final Evaluator<?> lhs, final Evaluator<?> rhs) {
+ super(lhs, rhs);
+ }
+
+ @Override
+ protected boolean compareNumbers(final Double lhs, final Double rhs) {
+ return lhs > rhs;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/GreaterThanOrEqualEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/GreaterThanOrEqualEvaluator.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/GreaterThanOrEqualEvaluator.java
new file mode 100644
index 0000000..d9da6f7
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/GreaterThanOrEqualEvaluator.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.query.evaluator.comparison;
+
+import org.apache.nifi.hl7.query.evaluator.Evaluator;
+
+public class GreaterThanOrEqualEvaluator extends AbstractNumericComparison {
+
+ public GreaterThanOrEqualEvaluator(final Evaluator<?> lhs, final Evaluator<?> rhs) {
+ super(lhs, rhs);
+ }
+
+ @Override
+ protected boolean compareNumbers(final Double lhs, final Double rhs) {
+ return lhs >= rhs;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/IsNullEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/IsNullEvaluator.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/IsNullEvaluator.java
new file mode 100644
index 0000000..0f16dd9
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/IsNullEvaluator.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.query.evaluator.comparison;
+
+import java.util.Collection;
+import java.util.Map;
+
+import org.apache.nifi.hl7.model.HL7Component;
+import org.apache.nifi.hl7.query.evaluator.BooleanEvaluator;
+import org.apache.nifi.hl7.query.evaluator.Evaluator;
+
+public class IsNullEvaluator extends BooleanEvaluator {
+
+ private final Evaluator<?> subjectEvaluator;
+
+ public IsNullEvaluator(final Evaluator<?> subjectEvaluator) {
+ this.subjectEvaluator = subjectEvaluator;
+ }
+
+ @Override
+ public Boolean evaluate(final Map<String, Object> objectMap) {
+ Object subjectValue = subjectEvaluator.evaluate(objectMap);
+ if (subjectValue == null) {
+ return true;
+ }
+
+ return isNull(subjectValue);
+ }
+
+ private boolean isNull(Object subjectValue) {
+ if (subjectValue == null) {
+ return true;
+ }
+
+ if (subjectValue instanceof HL7Component) {
+ subjectValue = ((HL7Component) subjectValue).getValue();
+ }
+
+ if (subjectValue instanceof Collection) {
+ final Collection<?> collection = (Collection<?>) subjectValue;
+ if (collection.isEmpty()) {
+ return true;
+ }
+
+ for (final Object obj : collection) {
+ if (!isNull(obj)) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ return subjectValue == null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/LessThanEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/LessThanEvaluator.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/LessThanEvaluator.java
new file mode 100644
index 0000000..2805860
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/LessThanEvaluator.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.query.evaluator.comparison;
+
+import org.apache.nifi.hl7.query.evaluator.Evaluator;
+
+public class LessThanEvaluator extends AbstractNumericComparison {
+
+ public LessThanEvaluator(final Evaluator<?> lhs, final Evaluator<?> rhs) {
+ super(lhs, rhs);
+ }
+
+ @Override
+ protected boolean compareNumbers(final Double lhs, final Double rhs) {
+ return lhs < rhs;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/LessThanOrEqualEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/LessThanOrEqualEvaluator.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/LessThanOrEqualEvaluator.java
new file mode 100644
index 0000000..e7eb156
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/LessThanOrEqualEvaluator.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.query.evaluator.comparison;
+
+import org.apache.nifi.hl7.query.evaluator.Evaluator;
+
+public class LessThanOrEqualEvaluator extends AbstractNumericComparison {
+
+ public LessThanOrEqualEvaluator(final Evaluator<?> lhs, final Evaluator<?> rhs) {
+ super(lhs, rhs);
+ }
+
+ @Override
+ protected boolean compareNumbers(final Double lhs, final Double rhs) {
+ return lhs <= rhs;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/NotEqualsEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/NotEqualsEvaluator.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/NotEqualsEvaluator.java
new file mode 100644
index 0000000..98715ff
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/NotEqualsEvaluator.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.query.evaluator.comparison;
+
+import org.apache.nifi.hl7.query.evaluator.Evaluator;
+
+public class NotEqualsEvaluator extends AbstractComparisonEvaluator {
+
+ public NotEqualsEvaluator(final Evaluator<?> lhs, final Evaluator<?> rhs) {
+ super(lhs, rhs);
+ }
+
+ @Override
+ protected boolean compare(final Object lhs, final Object rhs) {
+ return lhs != null && rhs != null && lhs != rhs && !lhs.equals(rhs) && !lhs.toString().equals(rhs.toString());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/NotEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/NotEvaluator.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/NotEvaluator.java
new file mode 100644
index 0000000..3343c1c
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/NotEvaluator.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.query.evaluator.comparison;
+
+import java.util.Map;
+
+import org.apache.nifi.hl7.query.evaluator.BooleanEvaluator;
+
+public class NotEvaluator extends BooleanEvaluator {
+
+ private final BooleanEvaluator subjectEvaluator;
+
+ public NotEvaluator(final BooleanEvaluator subjectEvaluator) {
+ this.subjectEvaluator = subjectEvaluator;
+ }
+
+ @Override
+ public Boolean evaluate(final Map<String, Object> objectMap) {
+ final Boolean subjectValue = subjectEvaluator.evaluate(objectMap);
+ return (subjectValue == null || Boolean.TRUE.equals(subjectValue));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/NotNullEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/NotNullEvaluator.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/NotNullEvaluator.java
new file mode 100644
index 0000000..80d6c45
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/NotNullEvaluator.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.query.evaluator.comparison;
+
+import java.util.Collection;
+import java.util.Map;
+
+import org.apache.nifi.hl7.model.HL7Component;
+import org.apache.nifi.hl7.query.evaluator.BooleanEvaluator;
+import org.apache.nifi.hl7.query.evaluator.Evaluator;
+
+public class NotNullEvaluator extends BooleanEvaluator {
+
+ private final Evaluator<?> subjectEvaluator;
+
+ public NotNullEvaluator(final Evaluator<?> subjectEvaluator) {
+ this.subjectEvaluator = subjectEvaluator;
+ }
+
+ @Override
+ public Boolean evaluate(final Map<String, Object> objectMap) {
+ Object subjectValue = subjectEvaluator.evaluate(objectMap);
+ if (subjectValue == null) {
+ return false;
+ }
+
+ return isNotNull(subjectValue);
+ }
+
+ private boolean isNotNull(Object subjectValue) {
+ if (subjectValue instanceof HL7Component) {
+ subjectValue = ((HL7Component) subjectValue).getValue();
+ }
+
+ if (subjectValue instanceof Collection) {
+ final Collection<?> collection = (Collection<?>) subjectValue;
+ if (collection.isEmpty()) {
+ return false;
+ }
+
+ for (final Object obj : collection) {
+ if (isNotNull(obj)) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ return subjectValue != null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/literal/IntegerLiteralEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/literal/IntegerLiteralEvaluator.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/literal/IntegerLiteralEvaluator.java
new file mode 100644
index 0000000..dfceee4
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/literal/IntegerLiteralEvaluator.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.query.evaluator.literal;
+
+import java.util.Map;
+
+import org.apache.nifi.hl7.query.evaluator.IntegerEvaluator;
+
+public class IntegerLiteralEvaluator extends IntegerEvaluator {
+
+ private final Integer value;
+
+ public IntegerLiteralEvaluator(final Integer value) {
+ this.value = value;
+ }
+
+ @Override
+ public Integer evaluate(final Map<String, Object> objectMap) {
+ return value;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/literal/StringLiteralEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/literal/StringLiteralEvaluator.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/literal/StringLiteralEvaluator.java
new file mode 100644
index 0000000..278ac1d
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/literal/StringLiteralEvaluator.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.query.evaluator.literal;
+
+import java.util.Map;
+
+import org.apache.nifi.hl7.query.evaluator.StringEvaluator;
+
+public class StringLiteralEvaluator extends StringEvaluator {
+
+ private final String value;
+
+ public StringLiteralEvaluator(final String value) {
+ this.value = value;
+ }
+
+ @Override
+ public String evaluate(final Map<String, Object> objectMap) {
+ return value;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/logic/AndEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/logic/AndEvaluator.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/logic/AndEvaluator.java
new file mode 100644
index 0000000..989cee4
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/logic/AndEvaluator.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.query.evaluator.logic;
+
+import java.util.Map;
+
+import org.apache.nifi.hl7.query.evaluator.BooleanEvaluator;
+
+public class AndEvaluator extends BooleanEvaluator {
+
+ private final BooleanEvaluator lhs;
+ private final BooleanEvaluator rhs;
+
+ public AndEvaluator(final BooleanEvaluator lhs, final BooleanEvaluator rhs) {
+ this.lhs = lhs;
+ this.rhs = rhs;
+ }
+
+ @Override
+ public Boolean evaluate(final Map<String, Object> objectMap) {
+ final Boolean lhsValue = lhs.evaluate(objectMap);
+ if (lhsValue == null || Boolean.FALSE.equals(lhsValue)) {
+ return false;
+ }
+
+ final Boolean rhsValue = rhs.evaluate(objectMap);
+ return (rhsValue != null && Boolean.TRUE.equals(rhsValue));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/logic/OrEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/logic/OrEvaluator.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/logic/OrEvaluator.java
new file mode 100644
index 0000000..e86daad
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/logic/OrEvaluator.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.query.evaluator.logic;
+
+import java.util.Map;
+
+import org.apache.nifi.hl7.query.evaluator.BooleanEvaluator;
+
+public class OrEvaluator extends BooleanEvaluator {
+
+ private final BooleanEvaluator lhs;
+ private final BooleanEvaluator rhs;
+
+ public OrEvaluator(final BooleanEvaluator lhs, final BooleanEvaluator rhs) {
+ this.lhs = lhs;
+ this.rhs = rhs;
+ }
+
+ @Override
+ public Boolean evaluate(final Map<String, Object> objectMap) {
+ final Boolean lhsValue = lhs.evaluate(objectMap);
+ if (lhsValue != null && Boolean.TRUE.equals(lhsValue)) {
+ return true;
+ }
+
+ final Boolean rhsValue = rhs.evaluate(objectMap);
+ return (rhsValue != null && Boolean.TRUE.equals(rhsValue));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/message/DeclaredReferenceEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/message/DeclaredReferenceEvaluator.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/message/DeclaredReferenceEvaluator.java
new file mode 100644
index 0000000..bb3f203
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/message/DeclaredReferenceEvaluator.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.query.evaluator.message;
+
+import java.util.Map;
+
+import org.apache.nifi.hl7.query.evaluator.Evaluator;
+import org.apache.nifi.hl7.query.evaluator.StringEvaluator;
+
+public class DeclaredReferenceEvaluator implements Evaluator<Object> {
+
+ private final StringEvaluator referenceNameEvaluator;
+
+ public DeclaredReferenceEvaluator(final StringEvaluator referenceNameEvaluator) {
+ this.referenceNameEvaluator = referenceNameEvaluator;
+ }
+
+ @Override
+ public Object evaluate(final Map<String, Object> objectMap) {
+ final String referenceName = referenceNameEvaluator.evaluate(objectMap);
+ return objectMap.get(referenceName);
+ }
+
+ @Override
+ public Class<? extends Object> getType() {
+ return Object.class;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/message/DotEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/message/DotEvaluator.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/message/DotEvaluator.java
new file mode 100644
index 0000000..238b540
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/message/DotEvaluator.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.query.evaluator.message;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.nifi.hl7.model.HL7Component;
+import org.apache.nifi.hl7.model.HL7Message;
+import org.apache.nifi.hl7.model.HL7Segment;
+import org.apache.nifi.hl7.query.evaluator.Evaluator;
+import org.apache.nifi.hl7.query.evaluator.IntegerEvaluator;
+
+public class DotEvaluator implements Evaluator<Object> {
+
+ private final Evaluator<?> lhs;
+ private final IntegerEvaluator rhs;
+
+ public DotEvaluator(final Evaluator<?> lhs, final IntegerEvaluator rhs) {
+ this.lhs = lhs;
+ this.rhs = rhs;
+ }
+
+ @Override
+ public Object evaluate(final Map<String, Object> objectMap) {
+ final Object lhsValue = this.lhs.evaluate(objectMap);
+ final Integer rhsValue = this.rhs.evaluate(objectMap);
+
+ if (lhsValue == null || rhsValue == null) {
+ return null;
+ }
+
+ final List<Object> results = new ArrayList<>();
+ if (lhsValue instanceof Collection) {
+ final Collection<?> lhsCollection = (Collection<?>) lhsValue;
+ for (final Object obj : lhsCollection) {
+ final Object val = getValue(obj, rhsValue);
+ results.add(val);
+ }
+ } else {
+ final Object val = getValue(lhsValue, rhsValue);
+ return val;
+ }
+
+ return results;
+ }
+
+ private Object getValue(final Object lhsValue, final int rhsValue) {
+ final List<?> list;
+ if (lhsValue instanceof HL7Message) {
+ list = ((HL7Message) lhsValue).getSegments();
+ } else if (lhsValue instanceof HL7Segment) {
+ list = ((HL7Segment) lhsValue).getFields();
+ } else if (lhsValue instanceof HL7Component) {
+ list = ((HL7Component) lhsValue).getComponents();
+ } else {
+ return null;
+ }
+
+ if (rhsValue > list.size()) {
+ return null;
+ }
+
+ // convert from 0-based to 1-based
+ return list.get(rhsValue - 1);
+ }
+
+ @Override
+ public Class<? extends Object> getType() {
+ return Object.class;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/message/FieldEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/message/FieldEvaluator.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/message/FieldEvaluator.java
new file mode 100644
index 0000000..bc8046e
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/message/FieldEvaluator.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.query.evaluator.message;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.nifi.hl7.model.HL7Field;
+import org.apache.nifi.hl7.model.HL7Segment;
+import org.apache.nifi.hl7.query.evaluator.Evaluator;
+import org.apache.nifi.hl7.query.evaluator.IntegerEvaluator;
+
+@SuppressWarnings("rawtypes")
+public class FieldEvaluator implements Evaluator<List> {
+
+ private final SegmentEvaluator segmentEvaluator;
+ private final IntegerEvaluator indexEvaluator;
+
+ public FieldEvaluator(final SegmentEvaluator segmentEvaluator, final IntegerEvaluator indexEvaluator) {
+ this.segmentEvaluator = segmentEvaluator;
+ this.indexEvaluator = indexEvaluator;
+ }
+
+ public List<HL7Field> evaluate(final Map<String, Object> objectMap) {
+ final List<HL7Segment> segments = segmentEvaluator.evaluate(objectMap);
+ if (segments == null) {
+ return Collections.emptyList();
+ }
+
+ final Integer index = indexEvaluator.evaluate(objectMap);
+ if (index == null) {
+ return Collections.emptyList();
+ }
+
+ final List<HL7Field> fields = new ArrayList<>();
+ for (final HL7Segment segment : segments) {
+ final List<HL7Field> segmentFields = segment.getFields();
+ if (segmentFields.size() <= index) {
+ continue;
+ }
+
+ fields.add(segmentFields.get(index));
+ }
+
+ return fields;
+ }
+
+ public Class<? extends List> getType() {
+ return List.class;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/message/MessageEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/message/MessageEvaluator.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/message/MessageEvaluator.java
new file mode 100644
index 0000000..f430b50
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/message/MessageEvaluator.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.query.evaluator.message;
+
+import java.util.Map;
+
+import org.apache.nifi.hl7.model.HL7Message;
+import org.apache.nifi.hl7.query.evaluator.Evaluator;
+
+public class MessageEvaluator implements Evaluator<HL7Message> {
+
+ public HL7Message evaluate(final Map<String, Object> objectMap) {
+ return (HL7Message) objectMap.get(Evaluator.MESSAGE_KEY);
+ }
+
+ public Class<? extends HL7Message> getType() {
+ return HL7Message.class;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/message/SegmentEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/message/SegmentEvaluator.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/message/SegmentEvaluator.java
new file mode 100644
index 0000000..9840461
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/message/SegmentEvaluator.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.query.evaluator.message;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.nifi.hl7.model.HL7Message;
+import org.apache.nifi.hl7.model.HL7Segment;
+import org.apache.nifi.hl7.query.evaluator.Evaluator;
+import org.apache.nifi.hl7.query.evaluator.StringEvaluator;
+
+@SuppressWarnings("rawtypes")
+public class SegmentEvaluator implements Evaluator<List> {
+
+ private final StringEvaluator segmentTypeEvaluator;
+
+ public SegmentEvaluator(final StringEvaluator segmentTypeEvaluator) {
+ this.segmentTypeEvaluator = segmentTypeEvaluator;
+ }
+
+ public List<HL7Segment> evaluate(final Map<String, Object> objectMap) {
+ final String segmentType = segmentTypeEvaluator.evaluate(objectMap);
+ if (segmentType == null) {
+ return Collections.emptyList();
+ }
+
+ final HL7Message message = (HL7Message) objectMap.get(Evaluator.MESSAGE_KEY);
+ final List<HL7Segment> segments = message.getSegments(segmentType);
+ return (segments == null) ? Collections.<HL7Segment>emptyList() : segments;
+ }
+
+ public Class<? extends List> getType() {
+ return List.class;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/exception/HL7QueryParsingException.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/exception/HL7QueryParsingException.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/exception/HL7QueryParsingException.java
new file mode 100644
index 0000000..e27dd58
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/exception/HL7QueryParsingException.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.query.exception;
+
+public class HL7QueryParsingException extends RuntimeException {
+
+ private static final long serialVersionUID = 1L;
+
+ public HL7QueryParsingException() {
+ super();
+ }
+
+ public HL7QueryParsingException(final Throwable cause) {
+ super(cause);
+ }
+
+ public HL7QueryParsingException(final String message) {
+ super(message);
+ }
+
+ public HL7QueryParsingException(final String message, final Throwable cause) {
+ super(message, cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/result/MissedResult.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/result/MissedResult.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/result/MissedResult.java
new file mode 100644
index 0000000..1043ab6
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/result/MissedResult.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.query.result;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.nifi.hl7.query.QueryResult;
+import org.apache.nifi.hl7.query.ResultHit;
+import org.apache.nifi.hl7.query.Selection;
+
+public class MissedResult implements QueryResult {
+
+ private final List<Selection> selections;
+
+ public MissedResult(final List<Selection> selections) {
+ this.selections = selections;
+ }
+
+ @Override
+ public List<String> getLabels() {
+ final List<String> labels = new ArrayList<>();
+ for (final Selection selection : selections) {
+ labels.add(selection.getName());
+ }
+ return labels;
+ }
+
+ @Override
+ public boolean isMatch() {
+ return false;
+ }
+
+ @Override
+ public ResultHit nextHit() {
+ return null;
+ }
+
+ @Override
+ public int getHitCount() {
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/result/StandardQueryResult.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/result/StandardQueryResult.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/result/StandardQueryResult.java
new file mode 100644
index 0000000..bcba697
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/result/StandardQueryResult.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.query.result;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.nifi.hl7.query.QueryResult;
+import org.apache.nifi.hl7.query.ResultHit;
+import org.apache.nifi.hl7.query.Selection;
+
+public class StandardQueryResult implements QueryResult {
+
+ private final List<Selection> selections;
+ private final Set<Map<String, Object>> hits;
+ private final Iterator<Map<String, Object>> hitIterator;
+
+ public StandardQueryResult(final List<Selection> selections, final Set<Map<String, Object>> hits) {
+ this.selections = selections;
+ this.hits = hits;
+
+ hitIterator = hits.iterator();
+ }
+
+ @Override
+ public boolean isMatch() {
+ return !hits.isEmpty();
+ }
+
+ @Override
+ public List<String> getLabels() {
+ final List<String> labels = new ArrayList<>();
+ for (final Selection selection : selections) {
+ labels.add(selection.getName());
+ }
+ return labels;
+ }
+
+ @Override
+ public int getHitCount() {
+ return hits.size();
+ }
+
+ @Override
+ public ResultHit nextHit() {
+ if (hitIterator.hasNext()) {
+ return new StandardResultHit(hitIterator.next());
+ } else {
+ return null;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/result/StandardResultHit.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/result/StandardResultHit.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/result/StandardResultHit.java
new file mode 100644
index 0000000..7fe9205
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/result/StandardResultHit.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.query.result;
+
+import java.util.Collections;
+import java.util.Map;
+
+import org.apache.nifi.hl7.query.ResultHit;
+
+public class StandardResultHit implements ResultHit {
+
+ private final Map<String, Object> values;
+
+ public StandardResultHit(final Map<String, Object> values) {
+ this.values = values;
+ }
+
+ @Override
+ public Object getValue(final String label) {
+ return values.get(label);
+ }
+
+ @Override
+ public Map<String, Object> getSelectedValues() {
+ return Collections.unmodifiableMap(values);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/test/java/org/apache/nifi/hl7/query/TestHL7Query.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/test/java/org/apache/nifi/hl7/query/TestHL7Query.java b/nifi-commons/nifi-hl7-query-language/src/test/java/org/apache/nifi/hl7/query/TestHL7Query.java
new file mode 100644
index 0000000..f78f82b
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/test/java/org/apache/nifi/hl7/query/TestHL7Query.java
@@ -0,0 +1,306 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.query;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.nifi.hl7.hapi.HapiMessage;
+import org.apache.nifi.hl7.model.HL7Field;
+import org.apache.nifi.hl7.model.HL7Message;
+import org.junit.Test;
+
+import ca.uhn.hl7v2.DefaultHapiContext;
+import ca.uhn.hl7v2.HL7Exception;
+import ca.uhn.hl7v2.HapiContext;
+import ca.uhn.hl7v2.model.Message;
+import ca.uhn.hl7v2.parser.PipeParser;
+import ca.uhn.hl7v2.validation.impl.ValidationContextFactory;
+
+@SuppressWarnings("resource")
+public class TestHL7Query {
+
+ @Test
+ public void testAssignAliases() {
+ final LinkedHashMap<String, List<Object>> possibleValueMap = new LinkedHashMap<>();
+
+ final List<Object> valuesA = new ArrayList<>();
+ valuesA.add("a");
+ valuesA.add("b");
+ valuesA.add("c");
+
+ final List<Object> valuesB = new ArrayList<>();
+ valuesB.add("d");
+
+ final List<Object> valuesC = new ArrayList<>();
+ valuesC.add("e");
+ valuesC.add("f");
+
+ final List<Object> valuesD = new ArrayList<>();
+ valuesD.add("g");
+ valuesD.add("h");
+
+ possibleValueMap.put("A", valuesA);
+ possibleValueMap.put("B", valuesB);
+ possibleValueMap.put("C", valuesC);
+ possibleValueMap.put("D", valuesD);
+
+ for (int i = 0; i < valuesA.size() * valuesB.size() * valuesC.size() * valuesD.size(); i++) {
+ System.out.println(i + " : " + HL7Query.assignAliases(possibleValueMap, i));
+ }
+
+ verifyAssignments(HL7Query.assignAliases(possibleValueMap, 0), "a", "d", "e", "g");
+ verifyAssignments(HL7Query.assignAliases(possibleValueMap, 1), "b", "d", "e", "g");
+ verifyAssignments(HL7Query.assignAliases(possibleValueMap, 2), "c", "d", "e", "g");
+ verifyAssignments(HL7Query.assignAliases(possibleValueMap, 3), "a", "d", "f", "g");
+ verifyAssignments(HL7Query.assignAliases(possibleValueMap, 4), "b", "d", "f", "g");
+ verifyAssignments(HL7Query.assignAliases(possibleValueMap, 5), "c", "d", "f", "g");
+ verifyAssignments(HL7Query.assignAliases(possibleValueMap, 6), "a", "d", "e", "h");
+ verifyAssignments(HL7Query.assignAliases(possibleValueMap, 7), "b", "d", "e", "h");
+ verifyAssignments(HL7Query.assignAliases(possibleValueMap, 8), "c", "d", "e", "h");
+ verifyAssignments(HL7Query.assignAliases(possibleValueMap, 9), "a", "d", "f", "h");
+ verifyAssignments(HL7Query.assignAliases(possibleValueMap, 10), "b", "d", "f", "h");
+ verifyAssignments(HL7Query.assignAliases(possibleValueMap, 11), "c", "d", "f", "h");
+ }
+
+ private void verifyAssignments(final Map<String, Object> map, final String a, final String b, final String c, final String d) {
+ assertEquals(a, map.get("A"));
+ assertEquals(b, map.get("B"));
+ assertEquals(c, map.get("C"));
+ assertEquals(d, map.get("D"));
+ }
+
+ @Test
+ public void testSelectMessage() throws HL7Exception, IOException {
+ final HL7Query query = HL7Query.compile("SELECT MESSAGE");
+ final HL7Message msg = createMessage(new File("src/test/resources/hypoglycemia"));
+ final QueryResult result = query.evaluate(msg);
+ assertTrue(result.isMatch());
+ final List<String> labels = result.getLabels();
+ assertEquals(1, labels.size());
+ assertEquals("MESSAGE", labels.get(0));
+
+ assertEquals(1, result.getHitCount());
+ assertEquals(msg, result.nextHit().getValue("MESSAGE"));
+ }
+
+ @Test
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ public void testSelectField() throws HL7Exception, IOException {
+ final HL7Query query = HL7Query.compile("SELECT PID.5");
+ final HL7Message msg = createMessage(new File("src/test/resources/hypoglycemia"));
+ final QueryResult result = query.evaluate(msg);
+ assertTrue(result.isMatch());
+ final List<String> labels = result.getLabels();
+ assertEquals(1, labels.size());
+ assertEquals(1, result.getHitCount());
+
+ final Object names = result.nextHit().getValue("PID.5");
+ assertTrue(names instanceof List);
+ final List<Object> nameList = (List) names;
+ assertEquals(1, nameList.size());
+ final HL7Field nameField = (HL7Field) nameList.get(0);
+ assertEquals("SMITH^JOHN", nameField.getValue());
+ }
+
+ @Test
+ public void testSelectAbnormalTestResult() throws HL7Exception, IOException {
+ final String query = "DECLARE result AS REQUIRED OBX SELECT result WHERE result.7 != 'N' AND result.1 = 1";
+
+ final HL7Query hl7Query = HL7Query.compile(query);
+ final QueryResult result = hl7Query.evaluate(createMessage(new File("src/test/resources/hypoglycemia")));
+ assertTrue(result.isMatch());
+ }
+
+ @Test
+ public void testFieldEqualsString() throws HL7Exception, IOException {
+ HL7Query hl7Query = HL7Query.compile("DECLARE result AS REQUIRED OBX SELECT MESSAGE WHERE result.7 = 'L'");
+ QueryResult result = hl7Query.evaluate(createMessage(new File("src/test/resources/hypoglycemia")));
+ assertTrue(result.isMatch());
+
+ hl7Query = HL7Query.compile("DECLARE result AS REQUIRED OBX SELECT MESSAGE WHERE result.7 = 'H'");
+ result = hl7Query.evaluate(createMessage(new File("src/test/resources/hypoglycemia")));
+ assertFalse(result.isMatch());
+ }
+
+ @Test
+ public void testLessThan() throws HL7Exception, IOException {
+ HL7Query hl7Query = HL7Query.compile("DECLARE result AS REQUIRED OBX SELECT MESSAGE WHERE result.4 < 600");
+ QueryResult result = hl7Query.evaluate(createMessage(new File("src/test/resources/hypoglycemia")));
+ assertTrue(result.isMatch());
+
+ hl7Query = HL7Query.compile("DECLARE result AS REQUIRED OBX SELECT MESSAGE WHERE result.4 < 59");
+ result = hl7Query.evaluate(createMessage(new File("src/test/resources/hypoglycemia")));
+ assertFalse(result.isMatch());
+ }
+
+ @Test
+ public void testCompareTwoFields() throws HL7Exception, IOException {
+ HL7Query hl7Query = HL7Query.compile("DECLARE result AS REQUIRED OBX SELECT MESSAGE WHERE result.4 < result.6.2");
+ QueryResult result = hl7Query.evaluate(createMessage(new File("src/test/resources/hypoglycemia")));
+ assertTrue(result.isMatch());
+
+ hl7Query = HL7Query.compile("DECLARE result AS REQUIRED OBX SELECT MESSAGE WHERE NOT(result.4 > result.6.3)");
+ result = hl7Query.evaluate(createMessage(new File("src/test/resources/hypoglycemia")));
+ assertFalse(result.isMatch());
+ }
+
+ @Test
+ public void testLessThanOrEqual() throws HL7Exception, IOException {
+ HL7Query hl7Query = HL7Query.compile("DECLARE result AS REQUIRED OBX SELECT MESSAGE WHERE result.4 <= 59");
+ QueryResult result = hl7Query.evaluate(createMessage(new File("src/test/resources/hypoglycemia")));
+ assertTrue(result.isMatch());
+
+ hl7Query = HL7Query.compile("DECLARE result AS REQUIRED OBX SELECT MESSAGE WHERE result.4 <= 600");
+ result = hl7Query.evaluate(createMessage(new File("src/test/resources/hypoglycemia")));
+ assertTrue(result.isMatch());
+
+ hl7Query = HL7Query.compile("DECLARE result AS REQUIRED OBX SELECT MESSAGE WHERE result.4 <= 58");
+ result = hl7Query.evaluate(createMessage(new File("src/test/resources/hypoglycemia")));
+ assertFalse(result.isMatch());
+ }
+
+ @Test
+ public void testGreaterThanOrEqual() throws HL7Exception, IOException {
+ HL7Query hl7Query = HL7Query.compile("DECLARE result AS REQUIRED OBX SELECT MESSAGE WHERE result.4 >= 59");
+ QueryResult result = hl7Query.evaluate(createMessage(new File("src/test/resources/hypoglycemia")));
+ assertTrue(result.isMatch());
+
+ hl7Query = HL7Query.compile("DECLARE result AS REQUIRED OBX SELECT MESSAGE WHERE result.4 >= 6");
+ result = hl7Query.evaluate(createMessage(new File("src/test/resources/hypoglycemia")));
+ assertTrue(result.isMatch());
+
+ hl7Query = HL7Query.compile("DECLARE result AS REQUIRED OBX SELECT MESSAGE WHERE result.4 >= 580");
+ result = hl7Query.evaluate(createMessage(new File("src/test/resources/hypoglycemia")));
+ assertFalse(result.isMatch());
+ }
+
+ @Test
+ public void testGreaterThan() throws HL7Exception, IOException {
+ HL7Query hl7Query = HL7Query.compile("DECLARE result AS REQUIRED OBX SELECT MESSAGE WHERE result.4 > 58");
+ QueryResult result = hl7Query.evaluate(createMessage(new File("src/test/resources/hypoglycemia")));
+ assertTrue(result.isMatch());
+
+ hl7Query = HL7Query.compile("DECLARE result AS REQUIRED OBX SELECT MESSAGE WHERE result.4 > 6");
+ result = hl7Query.evaluate(createMessage(new File("src/test/resources/hypoglycemia")));
+ assertTrue(result.isMatch());
+
+ hl7Query = HL7Query.compile("DECLARE result AS REQUIRED OBX SELECT MESSAGE WHERE result.4 > 580");
+ result = hl7Query.evaluate(createMessage(new File("src/test/resources/hypoglycemia")));
+ assertFalse(result.isMatch());
+ }
+
+ @Test
+ public void testDistinctValuesReturned() throws HL7Exception, IOException {
+ HL7Query hl7Query = HL7Query.compile("DECLARE result1 AS REQUIRED OBX, result2 AS REQUIRED OBX SELECT MESSAGE WHERE result1.7 = 'L' OR result2.7 != 'H'");
+ QueryResult result = hl7Query.evaluate(createMessage(new File("src/test/resources/hypoglycemia")));
+ assertTrue(result.isMatch());
+ assertEquals(1, result.getHitCount());
+ }
+
+ @Test
+ public void testAndWithParens() throws HL7Exception, IOException {
+ HL7Query hl7Query = HL7Query.compile("DECLARE result AS REQUIRED OBX SELECT MESSAGE WHERE result.7 = 'L' AND result.3.1 = 'GLU'");
+ QueryResult result = hl7Query.evaluate(createMessage(new File("src/test/resources/hypoglycemia")));
+ assertTrue(result.isMatch());
+
+ hl7Query = HL7Query.compile("DECLARE result AS REQUIRED OBX SELECT MESSAGE WHERE result.7 = 'L' AND result.3.1 = 'GLU'");
+ result = hl7Query.evaluate(createMessage(new File("src/test/resources/hyperglycemia")));
+ assertFalse(result.isMatch());
+
+ hl7Query = HL7Query.compile("DECLARE result AS REQUIRED OBX SELECT MESSAGE WHERE result.7 = 'H' AND result.3.1 = 'GLU'");
+ result = hl7Query.evaluate(createMessage(new File("src/test/resources/hypoglycemia")));
+ assertFalse(result.isMatch());
+
+ hl7Query = HL7Query.compile("DECLARE result AS REQUIRED OBX SELECT MESSAGE WHERE result.7 = 'H' AND result.3.1 = 'GLU'");
+ result = hl7Query.evaluate(createMessage(new File("src/test/resources/hyperglycemia")));
+ assertTrue(result.isMatch());
+
+ hl7Query = HL7Query.compile("DECLARE result AS REQUIRED OBX SELECT MESSAGE WHERE (result.7 = 'H') AND (result.3.1 = 'GLU')");
+ result = hl7Query.evaluate(createMessage(new File("src/test/resources/hyperglycemia")));
+ assertTrue(result.isMatch());
+
+ hl7Query = HL7Query.compile("DECLARE result AS REQUIRED OBX SELECT MESSAGE WHERE ((result.7 = 'H') AND (result.3.1 = 'GLU'))");
+ result = hl7Query.evaluate(createMessage(new File("src/test/resources/hyperglycemia")));
+ assertTrue(result.isMatch());
+
+ hl7Query = HL7Query.compile("DECLARE result AS REQUIRED OBX SELECT MESSAGE WHERE (( ((result.7 = 'H')) AND ( ((result.3.1 = 'GLU')) )))");
+ result = hl7Query.evaluate(createMessage(new File("src/test/resources/hyperglycemia")));
+ assertTrue(result.isMatch());
+
+ }
+
+ @Test
+ public void testIsNull() throws HL7Exception, IOException {
+ HL7Query hl7Query = HL7Query.compile("DECLARE result AS REQUIRED OBX SELECT MESSAGE WHERE result.999 IS NULL");
+ QueryResult result = hl7Query.evaluate(createMessage(new File("src/test/resources/hypoglycemia")));
+ assertTrue(result.isMatch());
+
+ hl7Query = HL7Query.compile("DECLARE result AS REQUIRED OBX SELECT MESSAGE WHERE result.1 IS NULL");
+ result = hl7Query.evaluate(createMessage(new File("src/test/resources/hypoglycemia")));
+ assertFalse(result.isMatch());
+
+ hl7Query = HL7Query.compile("SELECT MESSAGE WHERE ZZZ IS NULL");
+ result = hl7Query.evaluate(createMessage(new File("src/test/resources/hypoglycemia")));
+ assertTrue(result.isMatch());
+
+ hl7Query = HL7Query.compile("SELECT MESSAGE WHERE OBX IS NULL");
+ result = hl7Query.evaluate(createMessage(new File("src/test/resources/hypoglycemia")));
+ assertFalse(result.isMatch());
+ }
+
+ @Test
+ public void testNotNull() throws HL7Exception, IOException {
+ HL7Query hl7Query = HL7Query.compile("DECLARE result AS REQUIRED OBX SELECT MESSAGE WHERE result.999 NOT NULL");
+ QueryResult result = hl7Query.evaluate(createMessage(new File("src/test/resources/hypoglycemia")));
+ assertFalse(result.isMatch());
+
+ hl7Query = HL7Query.compile("DECLARE result AS REQUIRED OBX SELECT MESSAGE WHERE result.1 NOT NULL");
+ result = hl7Query.evaluate(createMessage(new File("src/test/resources/hypoglycemia")));
+ assertTrue(result.isMatch());
+
+ hl7Query = HL7Query.compile("SELECT MESSAGE WHERE ZZZ NOT NULL");
+ result = hl7Query.evaluate(createMessage(new File("src/test/resources/hypoglycemia")));
+ assertFalse(result.isMatch());
+
+ hl7Query = HL7Query.compile("SELECT MESSAGE WHERE OBX NOT NULL");
+ result = hl7Query.evaluate(createMessage(new File("src/test/resources/hypoglycemia")));
+ assertTrue(result.isMatch());
+ }
+
+ private HL7Message createMessage(final File file) throws HL7Exception, IOException {
+ final byte[] bytes = Files.readAllBytes(file.toPath());
+ final String msgText = new String(bytes, "UTF-8");
+
+ final HapiContext hapiContext = new DefaultHapiContext();
+ hapiContext.setValidationContext(ValidationContextFactory.noValidation());
+
+ final PipeParser parser = hapiContext.getPipeParser();
+ final Message message = parser.parse(msgText);
+ return new HapiMessage(message);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/test/resources/hyperglycemia
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/test/resources/hyperglycemia b/nifi-commons/nifi-hl7-query-language/src/test/resources/hyperglycemia
new file mode 100644
index 0000000..cae413b
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/test/resources/hyperglycemia
@@ -0,0 +1,5 @@
+MSH|^~\&|XXXXXX||HealthOrg01||||ORU^R01|Q1111111111111111111|P|2.3|
+PID|||000000001||SMITH^JOHN||19700101|M||||||||||999999999999|123456789|
+PD1||||1234567890^LAST^FIRST^M^^^^^NPI|
+OBR|1|341856649^HNAM_ORDERID|000000000000000000|648088^Basic Metabolic Panel|||20150101000100|||||||||1620^Johnson^John^R||||||20150101000100|||M|||||||||||20150101000100|
+OBX|1|NM|GLU^Glucose Lvl|159|mg/dL|65-99^65^99|H|||F|||20150101000100|
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/test/resources/hypoglycemia
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/test/resources/hypoglycemia b/nifi-commons/nifi-hl7-query-language/src/test/resources/hypoglycemia
new file mode 100644
index 0000000..764ffcc
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/test/resources/hypoglycemia
@@ -0,0 +1,5 @@
+MSH|^~\&|XXXXXX||HealthOrg01||||ORU^R01|Q1111111111111111111|P|2.3|
+PID|||000000001||SMITH^JOHN||19700101|M||||||||||999999999999|123456789|
+PD1||||1234567890^LAST^FIRST^M^^^^^NPI|
+OBR|1|341856649^HNAM_ORDERID|000000000000000000|648088^Basic Metabolic Panel|||20150101000100|||||||||1620^Johnson^John^R||||||20150101000100|||M|||||||||||20150101000100|
+OBX|1|NM|GLU^Glucose Lvl|59|mg/dL|65-99^65^99|L|||F|||20150101000100|
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-logging-utils/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-logging-utils/pom.xml b/nifi-commons/nifi-logging-utils/pom.xml
new file mode 100644
index 0000000..8f268ea
--- /dev/null
+++ b/nifi-commons/nifi-logging-utils/pom.xml
@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-commons</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>nifi-logging-utils</artifactId>
+ <description>Utilities for logging</description>
+ <dependencies>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-logging-utils/src/main/java/org/apache/nifi/logging/NiFiLog.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-logging-utils/src/main/java/org/apache/nifi/logging/NiFiLog.java b/nifi-commons/nifi-logging-utils/src/main/java/org/apache/nifi/logging/NiFiLog.java
new file mode 100644
index 0000000..3de8518
--- /dev/null
+++ b/nifi-commons/nifi-logging-utils/src/main/java/org/apache/nifi/logging/NiFiLog.java
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.logging;
+
+import org.slf4j.Logger;
+import org.slf4j.Marker;
+
+/**
+ *
+ */
+public class NiFiLog implements Logger {
+
+ private final Logger logger;
+
+ public NiFiLog(final Logger logger) {
+ this.logger = logger;
+ }
+
+ public Logger getWrappedLog() {
+ return logger;
+ }
+
+ @Override
+ public void warn(Marker marker, String string, Throwable thrwbl) {
+ if (logger.isDebugEnabled()) {
+ logger.warn(marker, string, thrwbl);
+ } else {
+ logger.warn(marker, string);
+ }
+ }
+
+ @Override
+ public void warn(Marker marker, String string, Object[] os) {
+ logger.warn(marker, string, os);
+ }
+
+ @Override
+ public void warn(Marker marker, String string, Object o, Object o1) {
+ logger.warn(marker, string, o, o1);
+ }
+
+ @Override
+ public void warn(Marker marker, String string, Object o) {
+ logger.warn(marker, string, o);
+ }
+
+ @Override
+ public void warn(Marker marker, String string) {
+ logger.warn(marker, string);
+ }
+
+ @Override
+ public void warn(String string, Throwable thrwbl) {
+ if (logger.isDebugEnabled()) {
+ logger.warn(string, thrwbl);
+ } else {
+ logger.warn(string);
+ }
+ }
+
+ @Override
+ public void warn(String string, Object o, Object o1) {
+ logger.warn(string, o, o1);
+ }
+
+ @Override
+ public void warn(String string, Object[] os) {
+ logger.warn(string, os);
+ }
+
+ @Override
+ public void warn(String string, Object o) {
+ logger.warn(string, o);
+ }
+
+ @Override
+ public void warn(String string) {
+ logger.warn(string);
+ }
+
+ @Override
+ public void trace(Marker marker, String string, Throwable thrwbl) {
+ logger.trace(marker, string, thrwbl);
+ }
+
+ @Override
+ public void trace(Marker marker, String string, Object[] os) {
+ logger.trace(marker, string, os);
+ }
+
+ @Override
+ public void trace(Marker marker, String string, Object o, Object o1) {
+ logger.trace(marker, string, o, o1);
+ }
+
+ @Override
+ public void trace(Marker marker, String string, Object o) {
+ logger.trace(marker, string, o);
+ }
+
+ @Override
+ public void trace(Marker marker, String string) {
+ logger.trace(marker, string);
+ }
+
+ @Override
+ public void trace(String string, Throwable thrwbl) {
+ logger.trace(string, thrwbl);
+ }
+
+ @Override
+ public void trace(String string, Object[] os) {
+ logger.trace(string, os);
+ }
+
+ @Override
+ public void trace(String string, Object o, Object o1) {
+ logger.trace(string, o, o1);
+ }
+
+ @Override
+ public void trace(String string, Object o) {
+ logger.trace(string, o);
+ }
+
+ @Override
+ public void trace(String string) {
+ logger.trace(string);
+ }
+
+ @Override
+ public boolean isWarnEnabled(Marker marker) {
+ return logger.isWarnEnabled(marker);
+ }
+
+ @Override
+ public boolean isWarnEnabled() {
+ return logger.isWarnEnabled();
+ }
+
+ @Override
+ public boolean isTraceEnabled(Marker marker) {
+ return logger.isTraceEnabled(marker);
+ }
+
+ @Override
+ public boolean isTraceEnabled() {
+ return logger.isTraceEnabled();
+ }
+
+ @Override
+ public boolean isInfoEnabled(Marker marker) {
+ return logger.isInfoEnabled(marker);
+ }
+
+ @Override
+ public boolean isInfoEnabled() {
+ return logger.isInfoEnabled();
+ }
+
+ @Override
+ public boolean isErrorEnabled(Marker marker) {
+ return logger.isErrorEnabled(marker);
+ }
+
+ @Override
+ public boolean isErrorEnabled() {
+ return logger.isErrorEnabled();
+ }
+
+ @Override
+ public boolean isDebugEnabled(Marker marker) {
+ return logger.isDebugEnabled(marker);
+ }
+
+ @Override
+ public boolean isDebugEnabled() {
+ return logger.isDebugEnabled();
+ }
+
+ @Override
+ public void info(Marker marker, String string, Throwable thrwbl) {
+ if (logger.isDebugEnabled()) {
+ logger.info(marker, string, thrwbl);
+ } else {
+ logger.info(marker, string);
+ }
+ }
+
+ @Override
+ public void info(Marker marker, String string, Object[] os) {
+ logger.info(marker, string, os);
+ }
+
+ @Override
+ public void info(Marker marker, String string, Object o, Object o1) {
+ logger.info(marker, string, o, o1);
+ }
+
+ @Override
+ public void info(Marker marker, String string, Object o) {
+ logger.info(marker, string, o);
+ }
+
+ @Override
+ public void info(Marker marker, String string) {
+ logger.info(marker, string);
+ }
+
+ @Override
+ public void info(String string, Throwable thrwbl) {
+ if (logger.isDebugEnabled()) {
+ logger.info(string, thrwbl);
+ } else {
+ logger.info(string);
+ }
+ }
+
+ @Override
+ public void info(String string, Object[] os) {
+ logger.info(string, os);
+ }
+
+ @Override
+ public void info(String string, Object o, Object o1) {
+ logger.info(string, o, o1);
+ }
+
+ @Override
+ public void info(String string, Object o) {
+ logger.info(string, o);
+ }
+
+ @Override
+ public void info(String string) {
+ logger.info(string);
+ }
+
+ @Override
+ public String getName() {
+ return logger.getName();
+ }
+
+ @Override
+ public void error(Marker marker, String string, Throwable thrwbl) {
+ if (logger.isDebugEnabled()) {
+ logger.error(marker, string, thrwbl);
+ } else {
+ logger.error(marker, string);
+ }
+ }
+
+ @Override
+ public void error(Marker marker, String string, Object[] os) {
+ logger.error(marker, string, os);
+ }
+
+ @Override
+ public void error(Marker marker, String string, Object o, Object o1) {
+ logger.error(marker, string, o, o1);
+ }
+
+ @Override
+ public void error(Marker marker, String string, Object o) {
+ logger.error(marker, string, o);
+ }
+
+ @Override
+ public void error(Marker marker, String string) {
+ logger.error(marker, string);
+ }
+
+ @Override
+ public void error(String string, Throwable thrwbl) {
+ if (logger.isDebugEnabled()) {
+ logger.error(string, thrwbl);
+ } else {
+ logger.error(string);
+ }
+ }
+
+ @Override
+ public void error(String string, Object[] os) {
+ logger.error(string, os);
+ }
+
+ @Override
+ public void error(String string, Object o, Object o1) {
+ logger.error(string, o, o1);
+ }
+
+ @Override
+ public void error(String string, Object o) {
+ logger.error(string, o);
+ }
+
+ @Override
+ public void error(String string) {
+ logger.error(string);
+ }
+
+ @Override
+ public void debug(Marker marker, String string, Throwable thrwbl) {
+ logger.debug(marker, string, thrwbl);
+ }
+
+ @Override
+ public void debug(Marker marker, String string, Object[] os) {
+ logger.debug(marker, string, os);
+ }
+
+ @Override
+ public void debug(Marker marker, String string, Object o, Object o1) {
+ logger.debug(marker, string, o, o1);
+ }
+
+ @Override
+ public void debug(Marker marker, String string, Object o) {
+ logger.debug(marker, string, o);
+ }
+
+ @Override
+ public void debug(Marker marker, String string) {
+ logger.debug(marker, string);
+ }
+
+ @Override
+ public void debug(String string, Throwable thrwbl) {
+ logger.debug(string, thrwbl);
+ }
+
+ @Override
+ public void debug(String string, Object[] os) {
+ logger.debug(string, os);
+ }
+
+ @Override
+ public void debug(String string, Object o, Object o1) {
+ logger.debug(string, o, o1);
+ }
+
+ @Override
+ public void debug(String string, Object o) {
+ logger.debug(string, o);
+ }
+
+ @Override
+ public void debug(String string) {
+ logger.debug(string);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-processor-utilities/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-processor-utilities/pom.xml b/nifi-commons/nifi-processor-utilities/pom.xml
new file mode 100644
index 0000000..2352c27
--- /dev/null
+++ b/nifi-commons/nifi-processor-utilities/pom.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-commons</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>nifi-processor-utils</artifactId>
+ <packaging>jar</packaging>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-utils</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-security-utils</artifactId>
+ </dependency>
+ </dependencies>
+</project>
[17/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/developer-guide.adoc
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/developer-guide.adoc b/nifi-docs/src/main/asciidoc/developer-guide.adoc
new file mode 100644
index 0000000..bce4917
--- /dev/null
+++ b/nifi-docs/src/main/asciidoc/developer-guide.adoc
@@ -0,0 +1,2170 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+NiFi Developer's Guide
+======================
+Apache NiFi Team <de...@nifi.apache.org>
+:homepage: http://nifi.apache.org
+
+
+== Introduction
+
+The intent of this Developer Guide is to provide the reader with the information needed to understand how Apache NiFi
+extensions are developed and help to explain the thought process behind developing the components. It provides an introduction to
+and explanation of the API that is used to develop extensions. It does not, however, go into great detail about each
+of the methods in the API, as this guide is intended to supplement the JavaDocs of the API rather than replace them.
+This guide also assumes that the reader is familiar with Java 7 and Apache Maven.
+
+This guide is written by developers for developers. It is expected that before reading this
+guide, you have a basic understanding of NiFi and the concepts of dataflow. If not, please see the link:overview.html[NiFi Overview]
+and the link:user-guide.html[NiFi User Guide] to familiarize yourself with the concepts of NiFi.
+
+
+[[components]]
+== NiFi Components
+
+NiFi provides several extension points to provide developers the
+ability to add functionality to the application to meet their needs. The following list provides a
+high-level description of the most common extension points:
+
+- Processor
+ * The Processor interface is the mechanism through which NiFi exposes access to
+ <<flowfile>>s, their attributes, and their content. The Processor is the basic building
+ block used to comprise a NiFi dataflow. This interface is used to accomplish
+ all of the following tasks:
+
+ ** Create FlowFiles
+ ** Read FlowFile content
+ ** Write FlowFile content
+ ** Read FlowFile attributes
+ ** Update FlowFile attributes
+ ** Ingest data
+ ** Egress data
+ ** Route data
+ ** Extract data
+ ** Modify data
+
+- ReportingTask
+ * The ReportingTask interface is a mechanism that NiFi exposes to allow metrics,
+ monitoring information, and internal NiFi state to be published to external
+ endpoints, such as log files, e-mail, and remote web services.
+
+- ControllerService
+ * A ControllerService provides shared state and functionality across Processors, other ControllerServices,
+ and ReportingTasks within a single JVM. An example use case may include loading a very
+ large dataset into memory. By performing this work in a ControllerService, the data
+ can be loaded once and be exposed to all Processors via this service, rather than requiring
+ many different Processors to load the dataset themselves.
+
+- FlowFilePrioritizer
+ * The FlowFilePrioritizer interface provides a mechanism by which <<flowfile>>s
+ in a queue can be prioritized, or sorted, so that the FlowFiles can be processed in an order
+ that is most effective for a particular use case.
+
+- AuthorityProvider
+ * An AuthorityProvide is responsible for determining which privileges and roles, if any,
+ a given user should be granted.
+
+
+[[processor_api]]
+== Processor API
+
+The Processor is the most widely used Component available in NiFi.
+Processors are the only Component
+to which access is given to create, remove, modify, or inspect
+FlowFiles (data and attributes).
+
+All Processors are loaded and instantiated using Java's ServiceLoader
+mechanism. This means that all
+Processors must adhere to the following rules:
+
+ - The Processor must have a default constructor.
+ - The Processor's JAR file must contain an entry in the META-INF/services directory named
+ `org.apache.nifi.processor.Processor`. This is a text file where each line contains the
+ fully-qualified class name of a Processor.
+
+While `Processor` is an interface that can be implemented directly, it
+will be extremely rare to do so, as
+the `org.apache.nifi.processor.AbstractProcessor` is the base class
+for almost all Processor implementations. The `AbstractProcessor` class provides a significant
+amount of functionality, which makes the task of developing a Processor much easier and more convenient.
+For the scope of this document, we will focus primarily on the `AbstractProcessor` class when dealing
+with the Processor API.
+
+.Concurrency Note
+NiFi is a highly concurrent framework. This means that all extensions
+must be thread-safe. If unfamiliar with writing concurrent software in Java, it is highly
+recommended that you familiarize yourself with the principles of Java concurrency.
+
+
+[[supporting_api]]
+=== Supporting API
+
+In order to understand the Processor API, we must first understand -
+at least at a high level - several supporting classes and interfaces, which are discussed below.
+
+[[flowfile]]
+==== FlowFile
+A FlowFile is a logical notion that correlates a piece of data with a
+set of Attributes about that data.
+Such attributes include a FlowFile's unique identifier, as well as its
+name, size, and any number of other
+flow-specific values. While the contents and attributes of a FlowFile
+can change, the FlowFile object is
+immutable. Modifications to a FlowFile are made possible by the ProcessSession.
+
+[[process_session]]
+==== ProcessSession
+The ProcessSession, often referred to as simply a "session," provides
+a mechanism by which FlowFiles can be created, destroyed, examined, cloned, and transferred to other
+Processors. Additionally, a ProcessSession provides mechanism for creating modified versions of
+FlowFiles, by adding or removing attributes, or by modifying the FlowFile's content. The ProcessSession
+also exposes a mechanism for emitting provenance events that provide for the ability to track the
+lineage and history of a FlowFile. After operations are performed on one or more FlowFiles, a
+ProcessSession can be either committed or rolled back.
+
+[[process_context]]
+==== ProcessContext
+The ProcessContext provides a bridge between a Processor and the framework. It provides information
+about how the Processor is currently configured and allows the Processor to perform
+Framework-specific tasks, such as yielding its resources so that the framework will schedule other
+Processors to run without consuming resources unnecessarily.
+
+
+[[property_descriptor]]
+==== PropertyDescriptor
+PropertyDescriptor defines a property that is to be used by a
+Processor, ReportingTask, or ControllerService.
+The definition of a property includes its name, a description of the
+property, an optional default value,
+validation logic, and an indicator as to whether or not the property
+is required in order for the Processor
+to be valid. PropertyDescriptors are created by instantiating an
+instance of the `PropertyDescriptor.Builder`
+class, calling the appropriate methods to fill in the details about
+the property, and finally calling
+the `build` method.
+
+
+[[validator]]
+==== Validator
+A PropertyDescriptor may specify one or more Validators that can be
+used to ensure that the user-entered value
+for a property is valid. If a Validator indicates that a property
+value is invalid, the Component will not be
+able to be run or used until the property becomes valid.
+
+
+[[validation_context]]
+==== ValidationContext
+When validating property values, a ValidationContext can be used to
+obtain ControllerServices,
+create PropertyValue objects, and compile and evaluate property values
+using the Expression Language.
+
+
+[[property_value]]
+==== PropertyValue
+All property values returned to a Processor are returned in the form
+of a PropertyValue object. This
+object has convenience methods for converting the value from a String
+to other forms, such as numbers
+and time periods, as well as providing an API for evaluating the
+Expression Language.
+
+
+[[relationship]]
+==== Relationship
+Relationships define the routes to which a FlowFile may be transfered
+from a Processor. Relationships
+are created by instantiating an instance of the `Relationship.Builder`
+class, calling the appropriate methods
+to fill in the details of the Relationship, and finally calling the
+`build` method.
+
+
+[[processor_initialization_context]]
+==== ProcessorInitializationContext
+After a Processor is created, its `initialize` method will be called
+with an `InitializationContext` object.
+This object exposes configuration to the Processor that will not
+change throughout the life of the Processor,
+such as the unique identifier of the Processor.
+
+[[ProcessorLog]]
+==== ProcessorLog
+Processors are encouraged to perform their logging via the
+`ProcessorLog` interface, rather than obtaining
+a direct instance of a third-party logger. This is because logging via
+the ProcessorLog allows the framework
+to render log messages that exceed s a configurable severity level to
+the User Interface, allowing those who
+monitor the dataflow to be notified when important events occur.
+Additionally, it provides a consistent logging
+format for all Processors by logging stack traces when in DEBUG mode
+and providing the Processor's unique
+identifier in log messages.
+
+
+
+
+
+[[AbstractProcessor]]
+=== AbstractProcessor API
+
+Since the vast majority of Processors will be created by extending the
+AbstractProcessor, it is the
+abstract class that we will examine in this section. The
+AbstractProcessor provides several methods that
+will be of interest to Processor developers.
+
+
+==== Processor Initialization
+
+When a Processor is created, before any other methods are invoked, the
+`init` method of the
+AbstractProcessor will be invoked. The method takes a single argument,
+which is of type
+`ProcessorInitializationContext`. The context object supplies the
+Processor with a ProcessorLog,
+the Processor's unique identifier, and a ControllerServiceLookup that
+can be used to interact with the
+configured ControllerServices. Each of these objects is stored by the
+AbstractProcessor and may be obtained by
+subclasses via the `getLogger`, `getIdentifier`, and
+`getControllerServiceLookup` methods, respectively.
+
+
+==== Exposing Processor's Relationships
+
+In order for a Processor to transfer a FlowFile to a new destination
+for follow-on processing, the
+Processor must first be able to expose to the Framework all of the
+Relationships that it currently supports.
+This allows users of the application to connect Processors to one
+another by creating
+Connections between Processors and assigning the appropriate
+Relationships to those Connections.
+
+A Processor exposes the valid set of Relationships by overriding the
+`getRelationships` method.
+This method takes no arguments and returns a `Set` of `Relationship`
+objects. For most Processors, this Set
+will be static, but other Processors will generate the Set
+dynamically, based on user configuration.
+For those Processors for which the Set is static, it is advisable to
+create an immutable Set in the Processor's
+constructor or init method and return that value, rather than
+dynamically generating the Set. This
+pattern lends itself to cleaner code and better performance.
+
+
+==== Exposing Processor Properties
+
+Most Processors will require some amount of user configuration before
+they are able to be used. The properties
+that a Processor supports are exposed to the Framework via the
+`getSupportedPropertyDescriptors` method.
+This method takes no arguments and returns a `List` of
+`PropertyDescriptor` objects. The order of the objects in the
+List is important in that it dictates the order in which the
+properties will be rendered in the User Interface.
+
+A `PropertyDescriptor` object is constructed by creating a new
+instance of the `PropertyDescriptor.Builder` object,
+calling the appropriate methods on the builder, and finally calling
+the `build` method.
+
+While this method covers most of the use cases, it is sometimes
+desirable to allow users to configure
+additional properties whose name are not known. This can be achieved
+by overriding the
+`getSupportedDynamicPropertyDescriptor` method. This method takes a
+`String` as its only argument, which
+indicates the name of the property. The method returns a
+`PropertyDescriptor` object that can be used to validate
+both the name of the property, as well as the value. Any
+PropertyDescriptor that is returned from this method
+should be built setting the value of `isDynamic` to true in the
+`PropertyDescriptor.Builder` class. The default
+behavior of AbstractProcessor is to not allow any dynamically created
+properties.
+
+
+==== Validating Processor Properties
+
+A Processor is not able to be started if its configuration is not
+valid. Validation of a Processor property can
+be achieved by setting a Validator on a PropertyDescriptor or by
+restricting the allowable values for a
+property via the PropertyDescriptor.Builder's `allowableValues` method
+or `identifiesControllerService` method.
+
+There are times, though, when validating a Processor's properties
+individually is not sufficient. For this purpose,
+the AbstractProcessor exposes a `customValidate` method. The method
+takes a single argument of type `ValidationContext`.
+The return value of this method is a `Collection` of
+`ValidationResult` objects that describe any problems that were
+found during validation. Only those ValidationResult objects whose
+`isValid` method returns `false` should be returned.
+This method will be invoked only if all properties are valid according
+to their associated Validators and Allowable Values.
+I.e., this method will be called only if all properties are valid
+in-and-of themselves, and this method allows for
+validation of a Processor's configuration as a whole.
+
+
+==== Responding to Changes in Configuration
+
+It is sometimes desirable to have a Processor eagerly react when its
+properties are changed. The `onPropertyModified`
+method allows a Processor to do just that. When a user changes the
+property values for a Processor, the
+`onPropertyModified` method will be called for each modified property.
+The method takes three arguments: the PropertyDescriptor that
+indicates which property was modified,
+the old value, and the new value. If the property had no previous
+value, the second argument will be `null`. If the property
+was removed, the third argument will be `null`. It is important to
+note that this method will be called regardless of whether
+or not the values are valid. This method will be called only when a
+value is actually modified, rather than being
+called when a user updates a Processor without changing its value. At
+the point that this method is invoked, it is guaranteed
+that the thread invoking this method is the only thread currently
+executing code in the Processor, unless the Processor itself
+creates its own threads.
+
+
+==== Performing the Work
+
+When a Processor has work to do, it is scheduled to do so by having
+its `onTrigger` method called by the framework.
+The method takes two arguments: a `ProcessContext` and a
+`ProcessSession`. The first step in the `onTrigger` method
+is often to obtain a FlowFile on which the work is to be performed by
+calling one of the `get` methods on the ProcessSession.
+For Processors that ingest data into NiFi from external sources, this
+step is skipped. The Processor is then free to examine
+FlowFile attributes; add, remove, or modify attributes; read or modify
+FlowFile content; and transfer FlowFiles to the appropriate
+Relationships.
+
+
+==== When Processors are Triggered
+
+A Processor's `onTrigger` method will be called only when it is
+scheduled to run and when work exists for the Processor.
+Work is said to exist for a Processor if any of the following conditions is met:
+
+- A Connection whose destination is the Processor has at least one
+FlowFile in its queue
+- The Processors has no incoming Connections
+- The Processor is annotated with the @TriggerWhenEmpty annotation
+
+Several factors exist that will contribute to when a Processor's
+`onTrigger` method is invoked. First, the Processor will not
+be triggered unless a user has configured the Processor to run. If a
+Processor is scheduled to run, the Framework periodically
+(the period is configured by users in the User Interface) checks if
+there is work for the Processor to do, as described above.
+If so, the Framework will check downstream destinations of the
+Processor. If any of the Processor's outbound Connections is full,
+by default, the Processor will not be scheduled to run.
+
+However, the `@TriggerWhenAnyDestinationAvailable` annotation may be
+added to the Processor's class. In this case, the requirement
+is changed so that only one downstream destination must be "available"
+(a destination is considered "available" if the Connection's
+queue is not full), rather than requiring that all downstream
+destinations be available.
+
+Also related to Processor scheduling is the `@TriggerSerially`
+annotation. Processors that use this Annotation will never have more
+than one thread running the `onTrigger` method simultaneously. It is
+crucial to note, though, that the thread executing the code
+may change from invocation to invocation. Therefore, care must still
+be taken to ensure that the Processor is thread-safe!
+
+
+
+=== Component Lifecycle
+
+The NiFi API provides lifecycle support through use of Java
+Annotations. The `org.apache.nifi.annotations.lifecycle` package
+contains
+several annotations for lifecycle management. The following
+Annotations may be applied to Java methods in a NiFi component to
+indicate to
+the framework when the methods should be called. For the discussion of
+Component Lifecycle, we will define a NiFi component as a
+Processor, ControllerServices, or ReportingTask.
+
+==== @OnAdded
+
+The `@OnAdded` annotation causes a method to be invoked as soon as a
+component is created. The
+component's `initialize` method (or `init` method, if subclasses
+`AbstractProcessor`) will be invoked after the component is
+constructed,
+followed by methods that are annotated with `@OnAdded`. If any method
+annotated with `@OnAdded` throws an Exception, an error will
+be returned to the user, and that component will not be added to the
+flow. Furthermore, other methods with this
+Annotation will not be invoked. This method will be called only once
+for the lifetime of a component.
+Methods with this Annotation must take zero arguments.
+
+
+==== @OnRemoved
+
+The `@OnRemoved` annotation causes a method to be invoked before a
+component is removed from the flow.
+This allows resources to be cleaned up before removing a component.
+Methods with this annotation must take zero arguments.
+If a method with this annotation throws an Exception, the component
+will still be removed.
+
+==== @OnScheduled
+
+This annotation indicates that a method should be called every time
+the component is scheduled to run. Because ControllerServices
+are not scheduled, using this annotation on a ControllerService does
+not make sense and will not be honored. It should be
+used only for Processors and Reporting Tasks. If any method with this
+annotation throws an Exception, other methods with this
+annotation will not be invoked, and a notification will be presented
+to the user. In this case, methods annotated with
+`@OnUnscheduled` are then triggered, followed by methods with the
+`@OnStopped` annotation (during this state, if any of these
+methods throws an Exception, those Exceptions are ignored). The
+component will then yield its execution for some period of time,
+referred to as the "Administrative Yield Duration," which is a value
+that is configured in the `nifi.properties` file. Finally, the
+process will start again, until all of the methods annotated with
+`@OnScheduled` have returned without throwing any Exception.
+Methods with this annotation may take zero arguments or may take a
+single argument. If the single argument variation is used,
+the argument must be of type `ProcessContext` if the component is a
+Processor or `ConfigurationContext` if the component
+is a ReportingTask.
+
+==== @OnUnscheduled
+
+Methods with this annotation will be called whenever a Processor or
+ReportingTask is no longer scheduled to run. At that time, many threads
+may still be active in the Processor's `onTrigger` method. If such a method
+throws an Exception, a log message will be generated, and the
+Exception will be otherwise
+ignored and other methods with this annotation will still be invoked.
+Methods with this annotation may take zero arguments or may take a
+single argument.
+If the single argument variation is used, the argument must be of type
+`ProcessContext` if the component is a Processor or
+`ConfigurationContext` if the
+component is a ReportingTask.
+
+
+==== @OnStopped
+
+Methods with this annotation will be called when a Processor or
+ReportingTask is no longer scheduled to run
+and all threads have returned from the `onTrigger` method. If such a
+method throws an Exception,
+a log message will be generated, and the Exception will otherwise be
+ignored; other methods with
+this annotation will still be invoked. Methods with this annotation
+must take zero arguments.
+
+
+==== @OnShutdown
+
+Any method that is annotated with the `@OnShutdown` annotation will be
+called when NiFi is successfully
+shut down. If such a method throws an Exception, a log message will be
+generated, and the
+Exception will be otherwise ignored and other methods with this
+annotation will still be invoked.
+Methods with this annotation must take zero arguments. Note: while
+NiFi will attempt to invoke methods
+with this annotation on all components that use it, this is not always
+possible. For example, the process
+may be killed unexpectedly, in which case it does not have a chance to
+invoke these methods. Therefore,
+while methods using this annotation can be used to clean up resources,
+for instance, they should not be
+relied upon for critical business logic.
+
+
+
+
+=== Reporting Processor Activity
+
+Processors are responsible for reporting their activity so that users
+are able to understand what happens
+to their data. Processors should log events via the ProcessorLog,
+which is accessible via the InitializationContext
+or by calling the `getLogger` method of `AbstractProcessor`.
+
+Additionally, Processors should use the `ProvenanceReporter`
+interface, obtained via the ProcessSession's
+`getProvenanceReporter` method. The ProvenanceReoprter should be used
+to indicate any time that content is
+received from an external source or sent to an external location. The
+ProvenanceReporter also has methods for
+reporting when a FlowFile is cloned, forked, or modified, and when
+multiple FlowFiles are merged into a single FlowFile
+as well as associating a FlowFile with some other identifier. However,
+these functions are less critical to report, as
+the framework is able to detect these things and emit appropriate
+events on the Processor's behalf. Yet, it is a best practice
+for the Processor developer to emit these events, as it becomes
+explicit in the code that these events are being emitted, and
+the developer is able to provide additional details to the events,
+such as the amount of time that the action took or
+pertinent information about the action that was taken. If the
+Processor emits an event, the framework will not emit a duplicate
+event. Instead, it always assumes that the Processor developer knows
+what is happening in the context of the Processor
+better than the framework does. The framework may, however, emit a
+different event. For example, if a Processor modifies both the
+content of a FlowFile and its attributes and then emits only an
+ATTRIBUTES_MODIFIED event, the framework will emit a CONTENT_MODIFIED
+event. The framework will not emit an ATTRIBUTES_MODIFIED event if any
+other event is emitted for that FlowFile (either by the
+Processor or the framework). This is due to the fact that all
+Provenance Events know about the attributes of the FlowFile before the
+event occurred as well as those attributes that occurred as a result
+of the processing of that FlowFile, and as a result the
+ATTRIBUTES_MODIFIED is generally considered redundant and would result
+in a rendering of the FlowFile lineage being very verbose.
+It is, however, acceptable for a Processor to emit this event along
+with others, if the event is considered pertinent from the
+perspective of the Processor.
+
+
+
+
+
+
+
+== Documenting a Component
+
+NiFi attempts to make the user experience as simple and convenient as
+possible by providing significant amount of documentation
+to the user from within the NiFi application itself via the User
+Interface. In order for this to happen, of course, Processor
+developers must provide that documentation to the framework. NiFi
+exposes a few different mechanisms for supplying documentation to
+the framework.
+
+
+=== Documenting Properties
+
+Individual properties can be documented by calling the `description`
+method of a PropertyDescriptor's builder as such:
+
+[source,java]
+----
+public static final PropertyDescriptor MY_PROPERTY = new PropertyDescriptor.Builder()
+ .name("My Property")
+ .description("Description of the Property")
+ ...
+ .build();
+----
+
+If the property is to provide a set of allowable values, those values
+are presented to the user in a drop-down field in the UI.
+Each of those values can also be given a description:
+
+[source,java]
+----
+public static final AllowableValue EXTENSIVE = new AllowableValue("Extensive", "Extensive",
+ "Everything will be logged - use with caution!");
+public static final AllowableValue VERBOSE = new AllowableValue("Verbose", "Verbose",
+ "Quite a bit of logging will occur");
+public static final AllowableValue REGULAR = new AllowableValue("Regular", "Regular",
+ "Typical logging will occur");
+
+public static final PropertyDescriptor LOG_LEVEL = new PropertyDescriptor.Builder()
+ .name("Amount to Log")
+ .description("How much the Processor should log")
+ .allowableValues(REGULAR, VERBOSE, EXTENSIVE)
+ .defaultValue(REGULAR.getValue())
+ ...
+ .build();
+----
+
+
+=== Documenting Relationships
+
+Processor Relationships are documented in much the same way that
+properties are - by calling the `description` method of a
+Relationship's builder:
+
+[source,java]
+----
+public static final Relationship MY_RELATIONSHIP = new Relationship.Builder()
+ .name("My Relationship")
+ .description("This relationship is used only if the Processor fails to process the data.")
+ .build();
+----
+
+
+=== Documenting Capability and Keywords
+
+The `org.apache.nifi.annotations.documentation` package provides Java
+annotations that can be used to document components. The
+CapabilityDescription
+annotation can be added to a Processor, Reporting Task, or Controller
+Service and is intended to provide a brief description of the
+functionality
+provided by the component. The Tags annotation has a `value` variable
+that is defined to be an Array of Strings. As such, it is used
+by providing multiple values as a comma-separated list of ++String++s
+with curly braces. These values are then incorporated into the UI by
+allowing
+users to filter the components based on a tag (i.e., a keyword).
+Additionally, the UI provides a tag cloud that allows users to select
+the tags that
+they want to filter by. The tags that are largest in the cloud are
+those tags that exist the most on the components in that instance of
+NiFi. An
+example of using these annotations is provided below:
+
+[source, java]
+----
+@Tags({"example", "documentation", "developer guide", "processor", "tags"})
+@CapabilityDescription("Example Processor that provides no real functionality but is provided" +
+ " for an example in the Developer Guide")
+public static final ExampleProcessor extends Processor {
+ ...
+}
+----
+
+=== Documenting FlowFile Attribute Interaction
+
+Many times a processor will expect certain FlowFile attributes be set on in-bound FlowFiles in order
+for the processor to function properly. In other cases a processor may update or
+create FlowFile attributes on the out-bound FlowFile. Processor developers may document both of these
+behaviors using the `ReadsAttribute` and `WritesAttribute` documentation annotations. These attributes are used to generate documentation
+that gives users a better understanding of how a processor will interact with the flow.
+
+Note: Because Java 7 does not support
+repeated annotations on a type, you may need to use `ReadsAttributes` and `WritesAttributes` to indicate
+that a processor reads or writes multiple FlowFile attributes. This annotation can only be applied to Processors. An example is listed below:
+
+[source, java]
+----
+@WritesAttributes({ @WritesAttribute(attribute = "invokehttp.status.code", description = "The status code that is returned"),
+ @WritesAttribute(attribute = "invokehttp.status.message", description = "The status message that is returned"),
+ @WritesAttribute(attribute = "invokehttp.response.body", description = "The response body"),
+ @WritesAttribute(attribute = "invokehttp.request.url", description = "The request URL"),
+ @WritesAttribute(attribute = "invokehttp.tx.id", description = "The transaction ID that is returned after reading the response"),
+ @WritesAttribute(attribute = "invokehttp.remote.dn", description = "The DN of the remote server") })
+public final class InvokeHTTP extends AbstractProcessor {
+----
+
+=== Documenting Related Components
+Often Processors and ControllerServices are related to one another. Sometimes its a put/get relation as in `PutFile` and `GetFile`.
+Sometimes a Processor uses a ControllerService like `InvokeHTTP` and `StandardSSLContextService`. Sometimes one ControllerService uses another
+like `DistributedMapCacheClientService` and `DistributedMapCacheServer`. Developers of these extension points may relate these
+different components using the `SeeAlso` tag. This annotation links these components in the documentation.
+`SeeAlso` can be applied to Processors, ControllerServices and ReportingTasks. An example of how to do this is listed below:
+
+[source, java]
+----
+@SeeAlso(GetFile.class)
+public class PutFile extends AbstractProcessor {
+----
+
+=== Advanced Documentation
+
+When the documentation methods above are not sufficient, NiFi provides
+the ability to expose more advanced documentation to the user via the
+"Usage" documentation. When a user right-clicks on a Processor, NiFi
+provides a "Usage" menu item in the context menu. Additionally, the
+UI exposes a "Help" link in the top-right corner, from which the same
+Usage information can be found.
+
+The advanced documentation of a Processor is provided as an HTML file named `additionalDetails.html`.
+This file should exist within a directory whose name is the
+fully-qualified
+name of the Processor, and this directory's parent should be named
+`docs` and exist in the root of the Processor's jar.
+This file will be linked from a generated HTML file that will contain
+all the Capability, Keyword, PropertyDescription and Relationship information,
+so it will not be necessary to duplicate that. This is a place
+to provide a rich explanation of what this Processor is doing, what kind of
+data it expects and produces, and what FlowFile attributes it expects and produces.
+Because this documentation is in an HTML format, you may include images and tables
+to best describe this component. The same methods can be used to provide advanced
+documentation for Processors, ControllerServices and ReportingTasks.
+
+
+
+== Common Processor Patterns
+
+While there are many different Processors available to NiFi users, the
+vast majority of them fall into
+one of several common design patterns. Below, we discuss these
+patterns, when the patterns are appropriate,
+reasons we follow these patterns, and things to watch out for when
+applying such patterns. Note that the patterns
+and recommendations discussed below are general guidelines and not
+hardened rules.
+
+
+[[ingress]]
+=== Data Ingress
+
+A Processor that ingests data into NiFi has a single Relationship
+names `success`. This Processor generates
+new FlowFiles via the ProcessSession `create` method and does not pull
+FlowFiles from incoming Connections.
+The Processor name starts with "Get" or "Listen," depending on whether
+it polls an external source or exposes
+some interface to which external sources can connect. The name ends
+with the protocol used for communications.
+Processors that follow this pattern include `GetFile`, `GetSFTP`,
+`ListenHTTP`, and `GetHTTP`.
+
+This Processor may create or initialize a Connection Pool in a method
+that uses the `@OnScheduled` annotation.
+However, because communications problems may prevent connections from
+being established or cause connections
+to be terminated, connections themselves are not created at this
+point. Rather, the connections are
+created or leased from the pool in the `onTrigger` method.
+
+The `onTrigger` method of this Processor begins by leasing a
+connection from the Connection Pool, if possible,
+or otherwise creates a connection to the external service. When no
+data is available from the
+external source, the `yield` method of the ProcessContext is called by
+the Processor and the method returns so
+that this Processor avoids continually running and depleting resources
+without benefit. Otherwise, this
+Processor then creates a FlowFile via the ProcessSession's `create`
+method and assigns an appropriate
+filename and path to the FlowFile (by adding the `filename` and `path`
+attributes), as well as any other
+attributes that may be appropriate. An OutputStream to the FlowFile's content is
+obtained via the ProcessSession's `write` method, passing a new
+OutputStreamCallback (which is usually
+an anonymous inner class). From within this callback, the Processor is
+able to write to the FlowFile and streams
+the content from the external resource to the FlowFile's OutputStream.
+If the desire is to write the entire contents
+of an InputStream to the FlowFile, the `importFrom` method of
+ProcessSession may be more convenient to use than the
+`write` method.
+
+When this Processor expects to receive many small files, it may be
+advisable to create several FlowFiles from a
+single session before committing the session. Typically, this allows
+the Framework to treat the content of the
+newly created FlowFiles much more efficiently.
+
+This Processor generates a Provenance event indicating that it has
+received data and specifies from
+where the data came. This Processor should log the creation of the
+FlowFile so that the FlowFile's
+origin can be determined by analyzing logs, if necessary.
+
+This Processor acknowledges receipt of the data and/or removes the
+data from the external source in order
+to prevent receipt of duplicate files. *This is done only after the
+ProcessSession by which the FlowFile was
+created has been committed!* Failure to adhere to this principle may
+result in data loss, as restarting NiFi
+before the session has been committed will result in the temporary
+file being deleted. Note, however, that it
+is possible using this approach to receive duplicate data because the
+application could be restarted after
+committing the session and before acknowledging or removing the data
+from the external source. In general, though,
+potential data duplication is preferred over potential data loss. The
+connection is finally returned or added to
+the Connection Pool, depending on whether the connection was leased
+from the Connection Pool to begin with or
+was created in the `onTrigger` method.
+
+If there is a communications problem, the connection is typically
+terminated and not returned (or added) to
+the Connection Pool. Connections to remote systems are torn down and
+the Connection Pool shutdown in a method
+annotated with the `@OnStopped` annotation so that resources can be reclaimed.
+
+
+=== Data Egress
+
+A Processor that publishes data to an external source has two
+Relationships: `success` and `failure`. The
+Processor name starts with "Put" followed by the protocol that is used
+for data transmission. Processors
+that follow this pattern include `PutEmail`, `PutSFTP`, and
+`PostHTTP` (note that the name does not
+begin with "Put" because this would lead to confusion, since PUT and
+POST have special meanings when dealing with
+HTTP).
+
+This Processor may create or initialize a Connection Pool in a method
+that uses the `@OnScheduled` annotation.
+However, because communications problems may prevent connections from
+being established or cause connections
+to be terminated, connections themselves are not created at this
+point. Rather, the connections are
+created or leased from the pool in the `onTrigger` method.
+
+The `onTrigger` method first obtains a FlowFile from the
+ProcessSession via the `get` method. If no FlowFile is
+available, the method returns without obtaining a connection to the
+remote resource.
+
+If at least one FlowFile is available, the Processor obtains a
+connection from the Connection Pool, if possible,
+or otherwise creates a new connection. If the Processor is neither
+able to lease a connection from the Connection Pool
+nor create a new connection, the FlowFile is routed to `failure`, the
+event is logged, and the method returns.
+
+If a connection was obtained, the Processor obtains an InputStream to
+the FlowFile's content by invoking the
+`read` method on the ProcessSession and passing an InputStreamCallback
+(which is often an anonymous inner class)
+and from within that callback transmits the contents of the FlowFile
+to the destination. The event is logged
+along with the amount of time taken to transfer the file and the data
+rate at which the file was transferred.
+A SEND event is reported to the ProvenanceReporter by obtaining the
+reporter from the ProcessSession via the
+`getProvenanceReporter` method and calling the `send` method on the
+reporter. The connection is returned or added
+to the Connection Pool, depending on whether the connection was leased
+from the pool or newly created by the
+`onTrigger` method.
+
+If there is a communications problem, the connection is typically
+terminated and not returned (or added) to
+the Connection Pool. If there is an issue sending the data to the
+remote resource, the desired approach for handling the
+error depends on a few considerations. If the issue is related to a
+network condition, the FlowFile is generally
+routed to `failure`. The FlowFile is not penalized because there is
+not necessary a problem with the data. Unlike the
+case of the <<ingress>> Processor, we typically do not call `yield` on
+the ProcessContext. This is because in the case of
+ingest, the FlowFile does not exist until the Processor is able to
+perform its function. However, in the case of a Put Processor,
+the DataFlow Manager may choose to route `failure` to a different
+Processor. This can allow for a "backup" system to be
+used in the case of problems with one system or can be used for load
+distribution across many systems.
+
+If a problem occurs that is data-related, one of two approaches should
+be taken. First, if the problem is likely to
+sort itself out, the FlowFile is penalized and then routed to
+`failure`. This is the case, for instance, with PutFTP,
+when a FlowFile cannot be transferred because of a file naming
+conflict. The presumption is that the file will eventually
+be removed from the directory so that the new file can be transferred.
+As a result, we penalize the FlowFile and route to
+`failure` so that we can try again later. In the other case, if there
+is an actual problem with the data (such as the data does
+not conform to some required specification), a different approach may
+be taken. In this case, it may be advantageous
+to break apart the `failure` relationship into a `failure` and a
+`communications failure` relationship. This allows the
+DataFlow Manager to determine how to handle each of these cases
+individually. It is important in these situations to document
+well the differences between the two Relationships by clarifying it in
+the "description" when creating the Relationship.
+
+Connections to remote systems are torn down and the Connection Pool
+shutdown in a method
+annotated with `@OnStopped` so that resources can be reclaimed.
+
+
+=== Route Based on Content (One-to-One)
+
+A Processor that routes data based on its content will take one of two
+forms: Route an incoming FlowFile to exactly
+one destination, or route incoming data to 0 or more destinations.
+Here, we will discuss the first case.
+
+This Processor has two relationships: `matched` and `unmatched`. If a
+particular data format is expected, the Processor
+will also have a `failure` relationship that is used when the input is
+not of the expected format. The Processor exposes
+a Property that indicates the routing criteria.
+
+If the Property that specifies routing criteria requires processing,
+such as compiling a Regular Expression, this processing
+is done in a method annotated with `@OnScheduled`, if possible. The
+result is then stored in a member variable that is marked
+as `volatile`.
+
+The `onTrigger` method obtains a single FlowFile. The method reads the
+contents of the FlowFile via the ProcessSession's `read`
+method, evaluating the Match Criteria as the data is streamed. The
+Processor then determines whether the FlowFile should be
+routed to `matched` or `unmatched` based on whether or not the
+criteria matched, and routes the FlowFile to the appropriate
+relationship.
+
+The Processor then emits a Provenance ROUTE event indicating which
+Relationship to which the Processor routed the FlowFile.
+
+This Processor is annotated with the `@SideEffectFree` and
+`@SupportsBatching` annotations from the `org.apache.nifi.annotations.behavior`
+package.
+
+
+=== Route Based on Content (One-to-Many)
+
+If a Processor will route a single FlowFile to potentially many
+relationships, this Processor will be slightly different than
+the above-described Processor for Routing Data Based on Content. This
+Processor typically has Relationships that are dynamically
+defined by the user as well as an `unmatched` relationship.
+
+In order for the user to be able to define additionally Properties,
+the `getSupportedDynamicPropertyDescriptor` method must be
+overridden. This method returns a PropertyDescriptor with the supplied
+name and an applicable Validator to ensure that the
+user-specified Matching Criteria is valid.
+
+In this Processor, the Set of Relationships that is returned by the
+`getRelationships` method is a member variable that is
+marked `volatile`. This Set is initially constructed with a single
+Relationship named `unmatched`. The `onPropertyModified` method
+is overridden so that when a Property is added or removed, a new
+Relationship is created with the same name. If the Processor has
+Properties that are not user-defined, it is important to check if the
+specified Property is user-defined. This can be achieved by
+calling the `isDynamic` method of the PropertyDescriptor that is
+passed to this method. If this Property is dynamic,
+a new Set of Relationships is then created, and the previous set of
+Relationships is copied into it. This new Set
+either has the newly created Relationship added to it or removed from
+it, depending on whether a new Property was added
+to the Processor or a Property was removed (Property removal is
+detected by check if the third argument to this function is `null`).
+The member variable holding the Set of Relationships is then updated
+to point to this new Set.
+
+If the Properties that specify routing criteria require processing,
+such as compiling a Regular Expression, this processing is done
+in a method annotated with `@OnScheduled`, if possible. The result is
+then stored in a member variable that is marked as `volatile`.
+This member variable is generally of type `Map` where the key is of
+type `Relationship` and the value's type is defined by the result of
+processing the property value.
+
+The `onTrigger` method obtains a FlowFile via the `get` method of
+ProcessSession. If no FlowFile is available, it returns immediately.
+Otherwise, a Set of type Relationship is created. The method reads the
+contents of the FlowFile via the ProcessSession's `read` method,
+evaluating each of the Match Criteria as the data is streamed. For any
+criteria that matches, the relationship associated with that Match
+Criteria is added to the Set of Relationships.
+
+After reading the contents of the FlowFile, the method checks if the
+Set of Relationships is empty. If so, the original FlowFile has
+an attribute added to it to indicate the Relationship to which it was
+routed and is routed to the `unmatched`. This is logged, a
+Provenance ROUTE event is emitted, and the method returns. If the size
+of the Set is equal to 1, the original FlowFile has an attribute
+added to it to indicate the Relationship to which it was routed and
+is routed to the Relationship specified by the entry in the Set.
+This is logged, a Provenance ROUTE event is emitted for the FlowFile,
+and the method returns.
+
+In the event that the Set contains more than 1 Relationship, the
+Processor creates a clone of the FlowFile for each Relationship,
+except
+for the first. This is done via the `clone` method of the
+ProcessSession. There is no need to report a CLONE Provenance Event,
+as the
+framework will handle this for you. The original FlowFile and each
+clone are routed to their appropriate Relationship with attribute
+indicating the name of the Relationship. A Provenance ROUTE event is
+emitted for each FlowFile. This is logged, and the method returns.
+
+This Processor is annotated with the `@SideEffectFree` and
+`@SupportsBatching` annotations from the
+`org.apache.nifi.annotations.behavior`
+package.
+
+
+=== Route Streams Based on Content (One-to-Many)
+
+The previous description of Route Based on Content (One-to-Many)
+provides an abstraction
+for creating a very powerful Processor. However, it assumes that each
+FlowFile will be routed
+in its entirety to zero or more Relationships. What if the incoming
+data format is a "stream" of
+many different pieces of information - and we want to send different
+pieces of this stream to
+different Relationships? For example, imagine that we want to have a
+RouteCSV Processor such that
+it is configured with multiple Regular Expressions. If a line in the
+CSV file matches a Regular
+Expression, that line should be included in the outbound FlowFile to
+the associated relationship.
+If a Regular Expression is associated with the Relationship
+"has-apples" and that Regular Expression
+matches 1,000 of the lines in the FlowFile, there should be one outbound
+FlowFile for the "has-apples" relationship that has 1,000 lines in it.
+If a different Regular Expression
+is associated with the Relationship "has-oranges" and that Regular
+Expression matches 50 lines in the
+FlowFile, there should be one outbound FlowFile for the "has-oranges"
+relationship that has 50 lines in it.
+I.e., one FlowFile comes in and two FlowFiles come out. The two
+FlowFiles may contain some of the same lines
+of text from the original FlowFile, or they may be entirely different.
+This is the type of Processor that
+we will discuss in this section.
+
+This Processor's name starts with "Route" and ends with the name of
+the data type that it routes. In our
+example here, we are routing CSV data, so the Processor is named
+RouteCSV. This Processor supports dynamic
+properties. Each user-defined property has a name that maps to the
+name of a Relationship. The value of
+the Property is in the format necessary for the "Match Criteria." In
+our example, the value of the property
+must be a valid Regular Expression.
+
+This Processor maintains an internal `ConcurrentMap` where the key is
+a `Relationship` and the value is of
+a type dependent on the format of the Match Criteria. In our example,
+we would maintain a
+`ConcurrentMap<Relationship, Pattern>`. This Processor overrides the
+`onPropertyModified` method.
+If the new value supplied to this method (the third argument) is null,
+the Relationship whose name is
+defined by the property name (the first argument) is removed from the
+ConcurrentMap. Otherwise, the new value
+is processed (in our example, by calling `Pattern.compile(newValue)`)
+and this value is added to the ConcurrentMap
+with the key again being the Relationship whose name is specified by
+the property name.
+
+This Processor will override the `customValidate` method. In this
+method, it will retrieve all Properties from
+the `ValidationContext` and count the number of PropertyDescriptors
+that are dynamic (by calling `isDynamic()`
+on the PropertyDescriptor). If the number of dynamic
+PropertyDescriptors is 0, this indicates that the user
+has not added any Relationships, so the Processor returns a
+`ValidationResult` indicating that the Processor
+is not valid because it has no Relationships added.
+
+The Processor returns all of the Relationships specified by the user
+when its `getRelationships` method is
+called and will also return an `unmatched` Relationship. Because this
+Processor will have to read and write to the
+Content Repository (which can be relatively expensive), if this
+Processor is expected to be used for very high
+data volumes, it may be advantageous to add a Property that allows the
+user to specify whether or not they care
+about the data that does not match any of the Match Criteria.
+
+When the `onTrigger` method is called, the Processor obtains a
+FlowFile via `ProcessSession.get`. If no data
+is available, the Processor returns. Otherwise, the Processor creates
+a `Map<Relationship, FlowFile>`. We will
+refer to this Map as `flowFileMap`. The Processor reads the incoming
+FlowFile by calling `ProcessSession.read`
+and provides an `InputStreamCallback`.
+From within the Callback, the Processor reads the first piece of data
+from the FlowFile. The Processor then
+evaluates each of the Match Criteria against this piece of data. If a
+particular criteria (in our example,
+a Regular Expression) matches, the Processor obtains the FlowFile from
+`flowFileMap` that belongs to the appropriate
+Relationship. If no FlowFile yet exists in the Map for this
+Relationship, the Processor creates a new FlowFile
+by calling `session.create(incomingFlowFile)` and then adds the new
+FlowFile to `flowFileMap`. The Processor then
+writes this piece of data to the FlowFile by calling `session.append`
+with an `OutputStreamCallback`. From within
+this OutputStreamCallback, we have access to the new FlowFile's
+OutputStream, so we are able to write the data
+to the new FlowFile. We then return from the OutputStreamCallback.
+After iterating over each of the Match Criteria,
+if none of them match, we perform the same routines as above for the
+`unmatched` relationship (unless the user
+configures us to not write out unmatched data). Now that we have
+called `session.append`, we have a new version of
+the FlowFile. As a result, we need to update our `flowFileMap` to
+associate the Relationship with the new FlowFile.
+
+If at any point, an Exception is thrown, we will need to route the
+incoming FlowFile to `failure`. We will also
+need to remove each of the newly created FlowFiles, as we won't be
+transferring them anywhere. We can accomplish
+this by calling `session.remove(flowFileMap.values())`. At this point,
+we will log the error and return.
+
+Otherwise, if all is successful, we can now iterate through the
+`flowFileMap` and transfer each FlowFile to the
+corresponding Relationship. The original FlowFile is then either
+removed or routed to an `original` relationship.
+For each of the newly created FlowFiles, we also emit a Provenance
+ROUTE event indicating which Relationship
+the FlowFile went to. It is also helpful to include in the details of
+the ROUTE event how many pieces of information
+were included in this FlowFile. This allows DataFlow Managers to
+easily see when looking at the Provenance
+Lineage view how many pieces of information went to each of the
+relationships for a given input FlowFile.
+
+Additionally, some Processors may need to "group" the data that is
+sent to each Relationship so that each FlowFile
+that is sent to a relationship has the same value. In our example, we
+may wan to allow the Regular Expression
+to have a Capturing Group and if two different lines in the CSV match
+the Regular Expression but have different
+values for the Capturing Group, we want them to be added to two
+different FlowFiles. The matching value could then
+be added to each FlowFile as an Attribute. This can be accomplished by
+modifying the `flowFileMap` such that
+it is defined as `Map<Relationship, Map<T, FlowFile>>` where `T` is
+the type of the Grouping Function (in our
+example, the Group would be a `String` because it is the result of
+evaluating a Regular Expression's
+Capturing Group).
+
+
+
+=== Route Based on Attributes
+
+This Processor is almost identical to the Route Data Based on Content
+Processors described above. It takes two different forms: One-to-One
+and
+One-to-Many, as do the Content-Based Routing Processors. This
+Processor, however, does not make any call to ProcessSession's `read`
+method,
+as it does not read FlowFile content. This Processor is typically very
+fast, so the `@SupportsBatching` annotation can be very important
+in this case.
+
+
+
+=== Split Content (One-to-Many)
+
+This Processor generally requires no user configuration, with the
+exception of the size of each Split to create. The `onTrigger` method
+obtains
+a FlowFile from its input queues. A List of type FlowFile is created.
+The original FlowFile is read via the ProcessSession's `read` method,
+and an InputStreamCallback is used. Within the InputStreamCallback,
+the content is read until a point is reached at which the FlowFile
+should be
+split. If no split is needed, the Callback returns, and the original
+FlowFile is routed to `success`. In this case, a Provenance ROUTE
+event
+is emitted. Typically, ROUTE events are not emitted when routing a
+FlowFile to `success` because this generates a very verbose lineage
+that
+becomes difficult to navigate. However, in this case,the event is
+useful because we would otherwise expect a FORK event and the absence
+of
+any event is likely to cause confusion. The fact that the FlowFile was
+not split but was instead transferred to `success` is logged, and the
+method returns.
+
+If a point is reached at which a FlowFile needs to be split, a new
+FlowFile is created via the ProcessSession's `create(FlowFile)` method
+or the
+`clone(FlowFile, long, long)` method. The next section of code depends
+on whether the `create` method is used or the `clone` method is used.
+Both methods are described below. Which solution is appropriate must
+be determined on a case-by-case basis.
+
+The Create Method is most appropriate when the data will not be
+directly copied from the original FlowFile to the new FlowFile.
+For example, if only some of the data will be copied, or if the data
+will be modified in some way before being copied to the new
+FlowFile, this method is necessary. However, if the content of the new
+FlowFile will be an exact copy of a portion of the original
+FlowFile, the Clone Method is much preferred.
+
+*Create Method*
+If using the `create` method, the method is called with the original
+FlowFile as the argument so that the newly created FlowFile will
+inherit
+the attributes of the original FlowFile and a Provenance FORK event
+will be created by the framework.
+
+The code then enters a `try/finally` block. Within the `finally`
+block, the newly created FlowFile is added to the List of FlowFiles
+that have
+been created. This is done within a `finally` block so that if an
+Exception is thrown, the newly created FlowFile will be appropriately
+cleaned up.
+Within the `try` block, the callback initiates a new callback by
+calling the ProcessSession's `write` method with an
+OutputStreamCallback.
+The appropriate data is then copied from the InputStream of the
+original FlowFile to the OutputStream for the new FlowFile.
+
+*Clone Method*
+If the content of the newly created created FlowFile is to be only a
+contiguous subset of the bytes of the original FlowFile, it is
+preferred
+to use the `clone(FlowFile, long, long)` method instead of the
+`create(FlowFile)` method of the ProcessSession. In this case, the
+offset
+of the original FlwoFile at which the new FlowFile's content should
+begin is passed as the second argument to the `clone` method. The
+length
+of the new FlowFile is passed as the third argument to the `clone`
+method. For example, if the original FlowFile was 10,000 bytes
+and we called `clone(flowFile, 500, 100)`, the FlowFile that would be
+returned to us would be identical to `flowFile` with respect to its
+attributes. However, the content of the newly created FlowFile would
+be 100 bytes in length and would start at offset 500 of the original
+FlowFile. That is, the contents of the newly created FlowFile would be
+the same as if you had copied bytes 500 through 599 of the original
+FlowFile.
+
+After the clone has been created, it is added to the List of FlowFiles.
+
+This method is much more highly preferred than the Create method, when
+applicable,
+because no disk I/O is required. The framework is able to simply
+create a new FlowFile
+that references a subset of the original FlowFile's content, rather
+than actually copying
+the data. However, this is not always possible. For example, if header
+information must be copied
+from the beginning of the original FlowFile and added to the beginning
+of each Split,
+then this method is not possible.
+
+
+*Both Methods*
+Regardless of whether the Clone Method or the Create Method is used,
+the following is applicable:
+
+If at any point in the InputStreamCallback, a condition is reached in
+which processing cannot continue
+(for example, the input is malformed), a `ProcessException` should be
+thrown. The call to the
+ProcesssSession's `read` method is wrapped in a `try/catch` block
+where `ProcessException` is
+caught. If an Exception is caught, a log message is generated
+explaining the error. The List of
+newly created FlowFiles is removed via the ProcessSession's `remove`
+method. The original FlowFile
+is routed to `failure`.
+
+If no problems arise, the original FlowFile is routed to `original`
+and all newly created FlowFiles
+are updated to include the following attributes:
+
+[options="header"]
+|===
+| Attribute Name | Description
+| `split.parent.uuid` | The UUID of the original FlowFile
+| `split.index` | A one-up number indicating which FlowFile in the list this is (the first FlowFile
+ created will have a value `0`, the second will have a value `1`, etc.)
+| `split.count` | The total number of split FlowFiles that were created
+|===
+
+The newly created FlowFiles are routed to `success`; this event is
+logged; and the method returns.
+
+
+=== Update Attributes Based on Content
+
+This Processor is very similar to the Route Based on Content
+Processors discussed above. Rather than
+routing a FlowFile to `matched` or `unmatched`, the FlowFile is
+generally routed to `success` or `failure`
+and attributes are added to the FlowFile as appropriate. The
+attributes to be added are configured in a
+manner similar to that of the Route Based on Content (One-to-Many),
+with the user defining their own
+properties. The name of the property indicates the name of an
+attribute to add. The value of the
+property indicates some Matching Criteria to be applied to the data.
+If the Matching Criteria matches
+the data, an attribute is added with the name the same as that of the
+Property. The value of the
+attribute is the criteria from the content that matched.
+
+For example, a Processor that evaluates XPath Expressions may allow
+user-defined XPaths to be
+entered. If the XPath matches the content of a FlowFile, that FlowFile
+will have an attribute added with
+the name being equal to that of the Property name and a value equal to
+the textual content of the XML Element or
+Attribute that matched the XPath. The `failure` relationship would
+then be used if the incoming FlowFile
+was not valid XML in this example. The `success` relationship would be
+used regardless of whether or not
+any matches were found. This can then be used to route the FlowFile
+when appropriate.
+
+This Processor emits a Provenance Event of type ATTRIBUTES_MODIFIED.
+
+
+=== Enrich/Modify Content
+
+The Enrich/Modify Content pattern is very common and very generic.
+This pattern is responsible for any
+general content modification. For the majority of cases, this
+Processor is marked with the
+`@SideEffectFree` and `@SupportsBatching` annotations. The Processor
+has any number of required and optional
+Properties, depending on the Processor's function. The Processor
+generally has a `success` and `failure` relationship.
+The `failure` relationship is generally used when the input file is
+not in the expected format.
+
+This Processor obtains a FlowFile and updates it using the
+ProcessSession's `write(StreamCallback)` method
+so that it is able to both read from the FlowFile's content and write
+to the next version of the FlowFile's
+content. If errors are encountered during the callback, the callback
+will throw a `ProcessException`. The
+call to the ProcessSession's `write` method is wrapped in a
+`try/catch` block that catches `ProcessException`
+and routes the FlowFile to failure.
+
+If the callback succeeds, a CONTENT_MODIFIED Provenance Event is emitted.
+
+
+
+== Error Handling
+
+When writing a Processor, there are several different unexpected cases that can occur.
+It is important that Processor developers understand the mechanics of how the NiFi framework
+behaves if Processors do not handle errors themselves, and it's important to understand
+what error handling is expected of Processors. Here, we will discuss how Processors should
+handle unexpected errors during the course of their work.
+
+
+=== Exceptions within the Processor
+
+During the execution of the `onTrigger` method of a Processor, many things can potentially go
+awry. Common failure conditions include:
+
+ - Incoming data is not in the expected format.
+ - Network connections to external services fail.
+ - Reading or writing data to a disk fails.
+ - There is a bug in the Processor or a dependent library.
+
+Any of these conditions can result in an Exception being thrown from the Processor. From the framework
+perspective, there are two types of Exceptions that can escape a Processor: `ProcessException` and
+all others.
+
+If a ProcessException is thrown from the Processor, the framework will assume that this is a failure that
+is a known outcome. Moreover, it is a condition where attempting to process the data again later may
+be successful. As a result, the framework will roll back the session that was being processed and penalize
+the FlowFiles that were being processed.
+
+If any other Exception escapes the Processor, though, the framework will assume that it is a failure that
+was not taken into account by the developer. In this case, the framework will also roll back the session
+and penalize the FlowFiles. However, in this case, we can get into some very problematic cases. For example,
+the Processor may be in a bad state and may continually run, depleting system resources, without providing
+any useful work. This is fairly common, for instance, when a NullPointerException is thrown continually.
+In order to avoid this case, if an Exception other than ProcessException is able to escape the Processor's
+`onTrigger` method, the framework will also "Administratively Yield" the Processor. This means that the
+Processor will not be triggered to run again for some amount of time. The amount of time is configured
+in the `nifi.properties` file but is 10 seconds by default.
+
+
+=== Exceptions within a callback: IOException, RuntimeException
+
+More often than not, when an Exception occurs in a Processor, it occurs from within a callback (I.e.,
+`InputStreamCallback`, `OutputStreamCallback`, or `StreamCallback`). That is, during the processing of a
+FlowFile's content. Callbacks are allowed to throw either `RuntimeException` or `IOException`. In the case
+of RuntimeException, this Exception will propagate back to the `onTrigger` method. In the case of an
+`IOException`, the Exception will be wrapped within a ProcessException and this ProcessException will then
+be thrown from the Framework.
+
+For this reason, it is recommended that Processors that use callbacks do so within a `try/catch` block
+and catch `ProcessException` as well as any other `RuntimeException` that they expect their callback to
+throw. It is *not* recommended that Processors catch the general `Exception` or `Throwable` cases, however.
+This is discouraged for two reasons.
+
+First, if an unexpected RuntimeException is thrown, it is likely a bug
+and allowing the framework to rollback the session will ensure no data loss and ensures that DataFlow Managers
+are able to deal with the data as they see fit by keeping the data queued up in place.
+
+Second, when an IOException is thrown from a callback, there really are two types of IOExceptions: those thrown
+from Processor code (for example, the data is not in the expected format or a network connection fails), and
+those that are thrown from the Content Repository (where the FlowFile content is stored). If the latter is the case,
+the framework will catch this IOException and wrap it into a `FlowFileAccessException`, which extends `RuntimeException`.
+This is done explicitly so that the Exception will escape the `onTrigger` method and the framework can handle this
+condition appropriately. Catching the general Exception prevents this from happening.
+
+
+=== Penalization vs. Yielding
+
+When an issue occurs during processing, the framework exposes two methods to allow Processor developers to avoid performing
+unnecessary work: "penalization" and "yielding." These two concepts can become confusing for developers new to the NiFi API.
+A developer is able to penalize a FlowFile by calling the `penalize(FlowFile)` method of ProcessSession. This causes the
+FlowFile itself to be inaccessible to downstream Processors for a period of time. The amount of time that the FlowFile is
+inaccessible is determined by the DataFlow Manager by setting the "Penalty Duration" setting in the Processor Configuration
+dialog. The default value is 30 seconds. Typically, this is done when a Processor determines that the data cannot be processed
+due to environmental reasons that are expected to sort themselves out. A great example of this is the PutSFTP processor, which
+will penalize a FlowFile if a file already exists on the SFTP server that has the same filename. In this case, the Processor
+penalizes the FlowFile and routes it to failure. A DataFlow Manager can then route failure back to the same PutSFTP Processor.
+This way, if a file exists with the same filename, the Processor will not attempt to send the file again for 30 seconds
+(or whatever period the DFM has configured the Processor to use). In the meantime, it is able to continue to process other
+FlowFiles.
+
+On the other hand, yielding allows a Processor developer to indicate to the framework that it will not be able to perform
+any useful function for some period of time. This commonly happens with a Processor that is communicating with a remote
+resource. If the Processor cannot connect to the remote resource, or if the remote resource is expected to provide data
+but reports that it has none, the Processor should call `yield` on the `ProcessContext` object and then return. By doing
+this, the Processor is telling the framework that it should not waste resources triggering this Processor to run, because
+there's nothing that it can do - it's better to use those resources to allow other Processors to run.
+
+
+=== Session Rollback
+
+Thus far, when we have discussed the `ProcessSession`, we have typically referred to it simply as a mechanism for accessing
+FlowFiles. However, it provides another very important capability, which is transactionality. All methods that are called
+on a ProcessSession happen as a transaction. When we decided to end the transaction, we can do so either by calling
+`commit()` or by calling `rollback()`. Typically, this is handled by the `AbstractProcessor` class: if the `onTrigger` method
+throws an Exception, the AbstractProcessor will catch the Exception, call `session.rollback()`, and then re-throw the Exception.
+Otherwise, the AbstractProcessor will call `commit()` on the ProcessSession.
+
+There are times, however, that developers will want to roll back a session explicitly. This can be accomplished at any time
+by calling the `rollback()` or `rollback(boolean)` method. If using the latter, the boolean indicates whether or not those
+FlowFiles that have been pulled from queues (via the ProcessSession `get` methods) should be penalized before being added
+back to their queues.
+
+When `rollback` is called, any modification that has occurred to the FlowFiles in that session are discarded, to included
+both content modification and attribute modification. Additionally, all Provenance Events are rolled back (with the exception
+of any SEND event that was emitted by passing a value of `true` for the `force` argument). The FlowFiles that were pulled from
+the input queues are then transferred back to the input queues (and optionally penalized) so that they can be processed again.
+
+On the other hand, when the `commit` method is called, the FlowFile's new state is persisted in the FlowFile Repository, and
+any Provenance Events that occurred are persisted in the Provenance Repository. The previous content is destroyed (unless
+another FlowFile references the same piece of content), and the FlowFiles are transferred to the outbound queues so that the
+next Processors can operate on the data.
+
+It is also important to note how this behavior is affected by using the `org.apache.nifi.annotations.behavior.SupportsBatching`
+annotation. If a Processor utilizes this annotation, calls to `ProcessSession.commit` may not take affect immediately. Rather,
+these commits may be batched together in order to provide higher throughput. However, if at any point, the Processor rolls back
+the ProcessSession, all changes since the last call to `commit` will be discarded and all "batched" commits will take affect.
+These "batched" commits are not rolled back.
+
+
+
+
+== General Design Considerations
+
+When designing a Processor, there are a few important design considering to keep in mind. This section of the Developer Guide
+brings to the forefront some of the ideas that a developer should be thinking about when creating a Processor.
+
+=== Consider the User
+
+One of the most important concepts to keep in mind when developing a Processor (or any other component) is the user
+experience that you are creating. It's important to remember that as the developer of such a component, you may have
+important knowledge about the context that others do not have. Documentation should always be supplied so that those
+less familiar with the process are able to use it with ease.
+
+When thinking about the user experience, it is also important to note that consistency is very important. It is best
+to stick with the standard <<naming-convensions>>. This is true for Processor names, Property names and value, Relationship
+names, and any other aspect that the user will experience.
+
+Simplicity is crucial! Avoid adding properties that you don't expect users to understand or change. As developers, we are
+told that hard-coding values is bad. But this sometimes results in developers exposing properties that, when asked for clarification,
+tell users to just leave the default value. This leads to confusion and complexity.
+
+
+=== Cohesion and Reusability
+
+For the sake of making a single, cohesive unit, developers are sometimes tempted to combine several functions into a single Processor.
+This is very true for the case when a Processor expects input data to be in format X so that the Processor can convert the data into
+format Y and send the newly-formatted data to some external service.
+
+Taking this approach of formatting the data for a particular endpoint and then sending the data to that endpoint within the same Processor
+has several drawbacks:
+
+ - The Processor becomes very complex, as it has to perform the data translation task as well as the task of
+ sending the data to the remote service.
+ - If the Processor is unable to communicate with the remote service, it will route the data to a `failure` Relationship. In this case,
+ the Processor will be responsible to perform the data translation again. And if it fails again, the translation is done yet again.
+ - If we have five different Processors that translate the incoming data into this new format before sending the data, we have a great
+ deal of duplicated code. If the schema changes, for instance, many Processors must be updated.
+ - This intermediate data is thrown away when the Processor finishes sending to the remote service. The intermediate data format
+ may well be useful to other Processors.
+
+In order to avoid these issues, and make Processors more reusable, a Processor should always stick to the principal of "do one thing and do
+it well." Such a Processor should be broken into two separate Processors: one to convert the data from Format X to Format Y, and another
+Processor to send data to the remote resource.
+
+
+[[naming-convensions]]
+=== Naming Conventions
+
+In order to deliver a consistent look and feel to users, it is advisable that Processors keep with standard naming conventions. The following
+is a list of standard conventions that are used:
+
+ - Processors that pull data from a remote system are named Get<Service> or Get<Protocol>, depending on if they poll data from arbitrary
+ sources over a known Protocol (such as GetHTTP or GetFTP) or if they pull data from a known service (such as GetKafka)
+ - Processors that push data to a remote system are named Put<Service> or Put<Protocol>.
+ - Relationship names are lower-cased and use spaces to delineated words.
+ - Property names capitalize significant words, as would be done with the title of a book.
+
+
+
+=== Processor Behavior Annotations
+
+When creating a Processor, the developer is able to provide hints to the framework about how to utilize the Processor most
+effectively. This is done by applying annotations to the Processor's class. The annotations that can be applied to a
+Processor exist in three sub-packages of `org.apache.nifi.annotations`. Those in the `documentation` sub-package are used
+to provide documentation to the user. Those in the `lifecycle` sub-package instruct the framework which methods should
+be called on the Processor in order to respond to the appropriate life-cycle events. Those in the `behavior` package
+help the framework understand how to interact with the Processor in terms of scheduling and general behavior.
+
+The following annotations from the `org.apache.nifi.annotations.behavior` package can be used to modify how the framework
+will handle your Processor:
+
+ - `EventDriven`: Instructs the framework that the Processor can be scheduled using the Event-Driven scheduling
+ strategy. This strategy is still experimental at this point, but can result in reduced resource utilization
+ on dataflows that do not handle extremely high data rates.
+
+ - `SideEffectFree`: Indicates that the Processor does not have any side effects external to NiFi. As a result, the
+ framework is free to invoke the Processor many times with the same input without causing any unexpected
+ results to occur. This implies idempotent behavior. This can be used by the framework to improve efficiency by
+ performing actions such as transferring a ProcessSession from one Processor to another, such that if
+ a problem occurs many Processors' actions can be rolled back and performed again.
+
+ - `SupportsBatching`: This annotation indicates that it is okay for the framework to batch together multiple
+ ProcessSession commits into a single commit. If this annotation is present, the user will be able to choose
+ whether they prefer high throughput or lower latency in the Processor's Scheduling tab. This annotation should
+ be applied to most Processors, but it comes with a caveat: if the Processor calls `ProcessSession.commit`,
+ there is no guarantee that the data has been safely stored in NiFi's Content, FlowFile, and Provenance Repositories.
+ As a result, it is not appropriate for those Processors that receive data from an external source, commit the session,
+ and then delete the remote data or confirm a transaction with a remote resource.
+
+ - `TriggerSerially`: When this annotation is present, the framework will not allow the user to schedule more than one
+ concurrent thread to execute the `onTrigger` method at a time. Instead, the number of thread ("Concurrent Tasks")
+ will always be set to `1`. This does *not*, however, mean that the Processor does not have to be thread-safe,
+ as the thread that is executing `onTrigger` may change between invocations.
+
+ - `TriggerWhenAnyDestinationAvailable`: By default, NiFi will not schedule a Processor to run if any of its outbound
+ queues is full. This allows back-pressure to be applied all the way a chain of Processors. However, some Processors
+ may need to run even if one of the outbound queues is full. This annotations indicates that the Processor should run
+ if any Relationship is "available." A Relationship is said to be "available" if none of the connections that use
+ that Relationship is full. For example, the DistributeLoad Processor makes use of this annotation. If the "round robin"
+ scheduling strategy is used, the Processor will not run if any outbound queue is full. However, if the "next available"
+ scheduling strategy is used, the Processor will run if any Relationship at all is available and will route FlowFiles
+ only to those relationships that are available.
+
+ - `TriggerWhenEmpty`: The default behavior is to trigger a Processor to run only if its input queue has at least one
+ FlowFile or if the Processor has no input queues (which is typical of a "source" Processor). Applying this annotation
+ will cause the framework to ignore the size of the input queues and trigger the Processor regardless of whether or
+ not there is any data on an input queue. This is useful, for example, if the Processor needs to be triggered to run
+ periodically to time out a network connection.
+
+
+=== Data Buffering
+
+An important point to keep in mind is that NiFi provides a generic data processing capability. Data can be in any format. Processors
+are generally scheduled with several threads. A common mistake that developers new to NiFi make is to buffer all the contents of a
+FlowFile in memory. While there are cases when this is required, it should be avoided if at all possible, unless it is well-known
+what format the data is in. For example, a Processor responsible for executing XPath against an XML document will need to load the
+entire contents of the data into memory. This is generally acceptable, as XML is not expected to be extremely large. However, a Processor
+that searches for a specific byte sequence may be used to search files that are hundreds of gigabytes or more. Attempting to load this
+into memory can cause a lot of problems - especially if multiple threads are processing different FlowFiles simultaneously.
+
+Instead of buffering this data into memory, it is advisable to instead evaluate the data as it is streamed from the Content Repository
+(i.e., scan the content from the `InputStream` that is provided to your callback by `ProcessSession.read`). Of course, in this case,
+we don't want to read from the Content Repository for each byte, so we would use a BufferedInputStream or somehow buffer some small
+amount of data, as appropriate.
+
+
+
+
+
+[[controller-services]]
+== Controller Services
+
+The `ControllerService` interface allows developers to share
+functionality and state across the JVM in a clean
+and consistent manner. The interface resembles that of the `Processor`
+interface but does not
+have an `onTrigger` method because Controller Services are not
+scheduled to run periodically, and
+Controller Services do not have Relationships because they are not
+integrated into the flow directly. Rather,
+they are used Processors, Reporting Tasks, and other Controller Services.
+
+[[developing-controller-service]]
+=== Developing a ControllerService
+
+Just like with the Processor interface, the ControllerService
+interface exposes methods for configuration,
+validation, and initialization. These methods are all identical to
+those of the Processor interface
+except that the `initialize` method is passed a
+`ControllerServiceInitializationContext`, rather
+than a `ProcessorInitializationContext`.
+
+Controller Services come with an additional constraint that Processors
+do not have. A Controller Service
+must be comprised of an interface that extends `ControllerService`.
+Implementations can then be interacted
+with only through their interface. A Processor, for instance, will
+never be given a concrete implementation of
+a ControllerService and therefore must reference the service only via
+interfaces that extends `ControllerService`.
+
+This constraint is in place mainly because a Processor can exist in
+one NiFi Archive (NAR) while the implementation
+of the Controller Service that the Processor lives in can exist in a
+different NAR. This is accomplished by
+the framework by dynamically implementing the exposed interfaces in
+such a way that the framework can
+switch to the appropriate ClassLoader and invoke the desired method on
+the concrete implementation. However,
+in order to make this work, the Processor and the Controller Service
+implementation must share the same definition
+of the Controller Service interface. Therefore, both of these NARs
+must depend on the NAR that houses the
+Controller Service's interface. See <<nars>> for more information.
+
+
+[[interacting-with-controller-service]]
+=== Interacting with a ControllerService
+
+ControllerServices may be obtained by a Processor, another
+ControllerService, or a ReportingTask
+by means of the ControllerServiceLookup or by using the
+`identifiesControllerService` method of the
+PropertyDescriptor's Builder class. The ControllerServiceLookup can be
+obtained by a Processor from the
+ProcessorInitializationContext that is passed to the `initialize`
+method. Likewise, it is obtained by
+a ControllerService from the ControllerServiceInitializationContext
+and by a ReportingTask via the
+ReportingConfiguration object passed to the `initialize` method.
+
+For most use cases, though, using the `identifiesControllerService`
+method of a PropertyDescriptor Builder
+is preferred and is the least complicated method. In order to use this
+method, we create a PropertyDescriptor
+that references a Controller Service as such:
+
+[source,java]
+----
+public static final PropertyDescriptor SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder()
+ .name("SSL Context Service")
+ .details("Specified the SSL Context Service that can be used to create secure connections")
+ .required(true)
+ .identifiesControllerService(SSLContextService.class)
+ .build();
+----
+
+Using this method, the user will be prompted to supply the SSL Context
+Service that should be used. This is
+done by providing the user with a drop-down menu from which they are
+able to choose any of the SSLContextService
+configurations that have been configured, regardless of the implementation.
+
+In order to make use of this service, the Processor can use code such as:
+
+[source,java]
+----
+final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT_SERVICE)
+ .asControllerService(SSLContextService.class);
+----
+
+Note here that `SSLContextService` is an interface that extends
+ControllerService. The only implementation
+at this time is the `StandardSSLContextService`. However, the
+Processor developer need not worry about this
+detail.
+
+
+
+
+
+
+
+== Reporting Tasks
+
+So far, we have mentioned little about how to convey to the outside
+world how NiFi and its components
+are performing. Is the system able to keep up with the incoming data
+rate? How much more can
+the system handle? How much data is processed at the peak time of day
+versus the least busy time of day?
+
+In order to answer these questions, and many more, NiFi provides a
+capability for reporting status,
+statistics, metrics, and monitoring information to external services
+by means of the `ReportingTask`
+interface. ReportingTasks are given access to a host of information to
+determine how the system is performing.
+
+
+=== Developing a Reporting Task
+
+Just like with the Processor and ControllerService interfaces, the
+ReportingTask interface exposes methods for
+configuration, validation, and initialization. These methods are all
+identical to those of the
+Processor and ControllerService interfaces except that the
+`initialize` method is passed a `ReportingConfiguration`
+object, as opposed to the initialization objects received by the other
+Components. The ReportingTask also has
+an `onTrigger` method that is invoked by the framework to trigger the
+task to perform its job.
+
+Within the `onTrigger` method, the ReportingTask is given access to a
+ReportingContext, from which configuration
+and information about the NiFi instance can be obtained. The
+BulletinRepository allows Bulletins to be queried
+and allows the ReportingTask to submit its own Bulletins, so that
+information will be rendered to users. The
+ControllerServiceLookup that is accessible via the Context provides
+access to ControllerServices that have been
+configured. However, this method of obtaining Controller Services is
+not the preferred method. Rather, the
+preferred method for obtaining a Controller Service is to reference
+the Controller Service in a PropertyDescriptor,
+as is discussed in the <<interacting-with-controller-service>> section.
+
+The `EventAccess` object that is exposed via the ReportingContext
+provides access to the `ProcessGroupStatus`,
+which exposes statistics about the amount of data processed in the
+past five minutes by Process Groups,
+Processors, Connections, and other Components. Additionally, the
+EventAccess object provides access to
+the ++ProvenanceEventRecord++s
+that have been stored in the `ProvenanceEventRepository`. These
+Provenance Events are emitted by Processors when
+data is received from external sources, emitted to external services,
+removed from the system, modified,
+or routed according to some decision that was made.
+
+Each ProvenanceEvent has the ID of the FlowFile, the type of Event,
+the creation time of the Event, and
+all FlowFile attributes associated with the FlowFile at the time that
+the FlowFile was accessed by the component
+as well as the FlowFile attributes that were associated with the
+FlowFile as a result of the processing that the
+event describes. This provides a great deal of information to
+ReportingTasks, allowing reports to be generated
+in many different ways to expose metrics and monitoring capabilities
+needed for any number of operational concerns.
+
+
+
+
+
+
+== Testing
+
+Testing the components that will be used within a larger framework can often be very cumbersome
+and tricky. With NiFi, we strive to make testing components as easy as possible. In order to do this,
+
<TRUNCATED>
[22/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/RingBuffer.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/RingBuffer.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/RingBuffer.java
new file mode 100644
index 0000000..8cadc01
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/RingBuffer.java
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * Thread-safe implementation of a RingBuffer
+ *
+ */
+public class RingBuffer<T> {
+
+ private final Object[] buffer;
+ private int insertionPointer = 0;
+ private boolean filled = false;
+
+ private final ReadWriteLock rwLock = new ReentrantReadWriteLock();
+ private final Lock readLock = rwLock.readLock();
+ private final Lock writeLock = rwLock.writeLock();
+
+ public RingBuffer(final int size) {
+ buffer = new Object[size];
+ }
+
+ /**
+ * Adds the given value to the RingBuffer and returns the value that was removed in order to make room.
+ *
+ * @param value the new value to add
+ * @return value previously in the buffer
+ */
+ @SuppressWarnings("unchecked")
+ public T add(final T value) {
+ Objects.requireNonNull(value);
+
+ writeLock.lock();
+ try {
+ final Object removed = buffer[insertionPointer];
+
+ buffer[insertionPointer] = value;
+
+ if (insertionPointer == buffer.length - 1) {
+ filled = true;
+ }
+
+ insertionPointer = (insertionPointer + 1) % buffer.length;
+ return (T) removed;
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ public int getSize() {
+ readLock.lock();
+ try {
+ return filled ? buffer.length : insertionPointer;
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ public List<T> getSelectedElements(final Filter<T> filter) {
+ return getSelectedElements(filter, Integer.MAX_VALUE);
+ }
+
+ public List<T> getSelectedElements(final Filter<T> filter, final int maxElements) {
+ final List<T> selected = new ArrayList<>(1000);
+ int numSelected = 0;
+ readLock.lock();
+ try {
+ for (int i = 0; i < buffer.length && numSelected < maxElements; i++) {
+ final int idx = (insertionPointer + i) % buffer.length;
+ final Object val = buffer[idx];
+ if (val == null) {
+ continue;
+ }
+
+ @SuppressWarnings("unchecked")
+ final T element = (T) val;
+ if (filter.select(element)) {
+ selected.add(element);
+ numSelected++;
+ }
+ }
+ } finally {
+ readLock.unlock();
+ }
+ return selected;
+ }
+
+ public int countSelectedElements(final Filter<T> filter) {
+ int numSelected = 0;
+ readLock.lock();
+ try {
+ for (int i = 0; i < buffer.length; i++) {
+ final int idx = (insertionPointer + i) % buffer.length;
+ final Object val = buffer[idx];
+ if (val == null) {
+ continue;
+ }
+
+ @SuppressWarnings("unchecked")
+ final T element = (T) val;
+ if (filter.select(element)) {
+ numSelected++;
+ }
+ }
+ } finally {
+ readLock.unlock();
+ }
+
+ return numSelected;
+ }
+
+ /**
+ * Removes all elements from the RingBuffer that match the given filter
+ *
+ * @param filter to use for deciding what is removed
+ * @return always zero
+ */
+ public int removeSelectedElements(final Filter<T> filter) {
+ int count = 0;
+
+ writeLock.lock();
+ try {
+ for (int i = 0; i < buffer.length; i++) {
+ final int idx = (insertionPointer + i + 1) % buffer.length;
+ final Object val = buffer[idx];
+ if (val == null) {
+ continue;
+ }
+
+ @SuppressWarnings("unchecked")
+ final T element = (T) val;
+
+ if (filter.select(element)) {
+ buffer[idx] = null;
+ }
+ }
+ } finally {
+ writeLock.unlock();
+ }
+
+ return count;
+ }
+
+ public List<T> asList() {
+ return getSelectedElements(new Filter<T>() {
+ @Override
+ public boolean select(final T value) {
+ return true;
+ }
+ });
+ }
+
+ public T getOldestElement() {
+ readLock.lock();
+ try {
+ return getElementData(insertionPointer);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ public T getNewestElement() {
+ readLock.lock();
+ try {
+ int index = (insertionPointer == 0) ? buffer.length - 1 : insertionPointer - 1;
+ return getElementData(index);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private T getElementData(final int index) {
+ readLock.lock();
+ try {
+ return (T) buffer[index];
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ /**
+ * Iterates over each element in the RingBuffer, calling the {@link ForEachEvaluator#evaluate(Object) evaluate} method on each element in the RingBuffer. If the Evaluator returns {@code false},
+ * the method will skip all remaining elements in the RingBuffer; otherwise, the next element will be evaluated until all elements have been evaluated.
+ *
+ * @param evaluator used to evaluate each item in the ring buffer
+ */
+ public void forEach(final ForEachEvaluator<T> evaluator) {
+ forEach(evaluator, IterationDirection.FORWARD);
+ }
+
+ /**
+ * Iterates over each element in the RingBuffer, calling the {@link ForEachEvaluator#evaluate(Object) evaluate} method on each element in the RingBuffer. If the Evaluator returns {@code false},
+ * the method will skip all remaining elements in the RingBuffer; otherwise, the next element will be evaluated until all elements have been evaluated.
+ *
+ * @param evaluator the evaluator
+ * @param iterationDirection the order in which to iterate over the elements in the RingBuffer
+ */
+ public void forEach(final ForEachEvaluator<T> evaluator, final IterationDirection iterationDirection) {
+ readLock.lock();
+ try {
+ final int startIndex;
+ final int endIndex;
+ final int increment;
+
+ if (iterationDirection == IterationDirection.FORWARD) {
+ startIndex = 0;
+ endIndex = buffer.length - 1;
+ increment = 1;
+ } else {
+ startIndex = buffer.length - 1;
+ endIndex = 0;
+ increment = -1;
+ }
+
+ for (int i = startIndex; (iterationDirection == IterationDirection.FORWARD ? i <= endIndex : i >= endIndex); i += increment) {
+ final int idx = (insertionPointer + i) % buffer.length;
+ final Object val = buffer[idx];
+ if (val == null) {
+ continue;
+ }
+
+ @SuppressWarnings("unchecked")
+ final T element = (T) val;
+ if (!evaluator.evaluate(element)) {
+ return;
+ }
+ }
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ public static interface Filter<S> {
+
+ boolean select(S value);
+ }
+
+ /**
+ * Defines an interface that can be used to iterate over all of the elements in the RingBuffer via the {@link #forEach} method
+ *
+ * @param <S> the type to evaluate
+ */
+ public static interface ForEachEvaluator<S> {
+
+ /**
+ * Evaluates the given element and returns {@code true} if the next element should be evaluated, {@code false} otherwise
+ *
+ * @param value the value to evaluate
+ * @return true if should continue evaluating; false otherwise
+ */
+ boolean evaluate(S value);
+ }
+
+ public static enum IterationDirection {
+
+ FORWARD,
+ BACKWARD;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/StopWatch.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/StopWatch.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/StopWatch.java
new file mode 100644
index 0000000..cffe49c
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/StopWatch.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import java.util.concurrent.TimeUnit;
+
+public final class StopWatch {
+
+ private long startNanos = -1L;
+ private long duration = -1L;
+
+ /**
+ * Creates a StopWatch but does not start it
+ */
+ public StopWatch() {
+ this(false);
+ }
+
+ /**
+ * @param autoStart whether or not the timer should be started automatically
+ */
+ public StopWatch(final boolean autoStart) {
+ if (autoStart) {
+ start();
+ }
+ }
+
+ public void start() {
+ this.startNanos = System.nanoTime();
+ this.duration = -1L;
+ }
+
+ public void stop() {
+ if (startNanos < 0) {
+ throw new IllegalStateException("StopWatch has not been started");
+ }
+ this.duration = System.nanoTime() - startNanos;
+ this.startNanos = -1L;
+ }
+
+ /**
+ * Returns the amount of time that the StopWatch was running.
+ *
+ * @param timeUnit the unit for which the duration should be reported
+ * @return the duration of the stopwatch in the specified unit
+ *
+ * @throws IllegalStateException if the StopWatch has not been stopped via {@link #stop()}
+ */
+ public long getDuration(final TimeUnit timeUnit) {
+ if (duration < 0) {
+ throw new IllegalStateException("Cannot get duration until StopWatch has been stopped");
+ }
+ return timeUnit.convert(duration, TimeUnit.NANOSECONDS);
+ }
+
+ /**
+ * Returns the amount of time that has elapsed since the timer was started.
+ *
+ * @param timeUnit the unit for which the elapsed time should be computed
+ * @return the elapsed time in the specified unit
+ */
+ public long getElapsed(final TimeUnit timeUnit) {
+ return timeUnit.convert(System.nanoTime() - startNanos, TimeUnit.NANOSECONDS);
+ }
+
+ public String calculateDataRate(final long bytes) {
+ final double seconds = (double) duration / 1000000000.0D;
+ final long dataSize = (long) (bytes / seconds);
+ return FormatUtils.formatDataSize(dataSize) + "/sec";
+ }
+
+ public String getDuration() {
+ final StringBuilder sb = new StringBuilder();
+
+ long duration = this.duration;
+ final long minutes = (duration > 60000000000L) ? (duration / 60000000000L) : 0L;
+ duration -= TimeUnit.NANOSECONDS.convert(minutes, TimeUnit.MINUTES);
+
+ final long seconds = (duration > 1000000000L) ? (duration / 1000000000L) : 0L;
+ duration -= TimeUnit.NANOSECONDS.convert(seconds, TimeUnit.SECONDS);
+
+ final long millis = (duration > 1000000L) ? (duration / 1000000L) : 0L;
+ duration -= TimeUnit.NANOSECONDS.convert(millis, TimeUnit.MILLISECONDS);
+
+ final long nanos = duration % 1000000L;
+
+ if (minutes > 0) {
+ sb.append(minutes).append(" minutes");
+ }
+
+ if (seconds > 0) {
+ if (minutes > 0) {
+ sb.append(", ");
+ }
+
+ sb.append(seconds).append(" seconds");
+ }
+
+ if (millis > 0) {
+ if (seconds > 0) {
+ sb.append(", ");
+ }
+
+ sb.append(millis).append(" millis");
+ }
+ if (seconds == 0 && millis == 0) {
+ sb.append(nanos).append(" nanos");
+ }
+
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/Tuple.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/Tuple.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/Tuple.java
new file mode 100644
index 0000000..c797c7f
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/Tuple.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+public class Tuple<A, B> {
+
+ final A key;
+ final B value;
+
+ public Tuple(A key, B value) {
+ this.key = key;
+ this.value = value;
+ }
+
+ public A getKey() {
+ return key;
+ }
+
+ public B getValue() {
+ return value;
+ }
+
+ @Override
+ public boolean equals(final Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other == this) {
+ return true;
+ }
+ if (!(other instanceof Tuple)) {
+ return false;
+ }
+
+ final Tuple<?, ?> tuple = (Tuple<?, ?>) other;
+ if (key == null) {
+ if (tuple.key != null) {
+ return false;
+ }
+ } else {
+ if (!key.equals(tuple.key)) {
+ return false;
+ }
+ }
+
+ if (value == null) {
+ if (tuple.value != null) {
+ return false;
+ }
+ } else {
+ if (!value.equals(tuple.value)) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return 581 + (this.key == null ? 0 : this.key.hashCode()) + (this.value == null ? 0 : this.value.hashCode());
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/concurrency/DebugDisabledTimedLock.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/concurrency/DebugDisabledTimedLock.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/concurrency/DebugDisabledTimedLock.java
new file mode 100644
index 0000000..8faf3ba
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/concurrency/DebugDisabledTimedLock.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.concurrency;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+
+public class DebugDisabledTimedLock implements DebuggableTimedLock {
+
+ private final Lock lock;
+
+ public DebugDisabledTimedLock(final Lock lock) {
+ this.lock = lock;
+ }
+
+ /**
+ *
+ * @return true if lock obtained; false otherwise
+ */
+ @Override
+ public boolean tryLock() {
+ return lock.tryLock();
+ }
+
+ /**
+ *
+ * @param timeout the duration of time to wait for the lock
+ * @param timeUnit the unit which provides meaning to the duration
+ * @return true if obtained lock in time; false otherwise
+ */
+ @Override
+ public boolean tryLock(final long timeout, final TimeUnit timeUnit) {
+ try {
+ return lock.tryLock(timeout, timeUnit);
+ } catch (InterruptedException e) {
+ return false;
+ }
+ }
+
+ @Override
+ public void lock() {
+ lock.lock();
+ }
+
+ @Override
+ public void unlock(final String task) {
+ lock.unlock();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/concurrency/DebugEnabledTimedLock.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/concurrency/DebugEnabledTimedLock.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/concurrency/DebugEnabledTimedLock.java
new file mode 100644
index 0000000..e7d599e
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/concurrency/DebugEnabledTimedLock.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.concurrency;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DebugEnabledTimedLock implements DebuggableTimedLock {
+
+ private final Lock lock;
+ private final Logger logger;
+ private long lockTime = 0L;
+
+ private final Map<String, Long> lockIterations = new HashMap<>();
+ private final Map<String, Long> lockNanos = new HashMap<>();
+
+ private final String name;
+ private final int iterationFrequency;
+
+ public DebugEnabledTimedLock(final Lock lock, final String name, final int iterationFrequency) {
+ this.lock = lock;
+ this.name = name;
+ this.iterationFrequency = iterationFrequency;
+ logger = LoggerFactory.getLogger(TimedLock.class.getName() + "." + name);
+ }
+
+ /**
+ * @return true if lock obtained; false otherwise
+ */
+ @Override
+ public boolean tryLock() {
+ logger.trace("Trying to obtain Lock: {}", name);
+ final boolean success = lock.tryLock();
+ if (!success) {
+ logger.trace("TryLock failed for Lock: {}", name);
+ return false;
+ }
+ logger.trace("TryLock successful");
+
+ return true;
+ }
+
+ /**
+ * @param timeout duration to wait for lock
+ * @param timeUnit unit to understand given duration
+ * @return true if lock obtained in time; false otherwise
+ */
+ @Override
+ public boolean tryLock(final long timeout, final TimeUnit timeUnit) {
+ logger.trace("Trying to obtain Lock {} with a timeout of {} {}", name, timeout, timeUnit);
+ final boolean success;
+ try {
+ success = lock.tryLock(timeout, timeUnit);
+ } catch (final InterruptedException ie) {
+ return false;
+ }
+
+ if (!success) {
+ logger.trace("TryLock failed for Lock {} with a timeout of {} {}", name, timeout, timeUnit);
+ return false;
+ }
+ logger.trace("TryLock successful");
+ return true;
+ }
+
+ @Override
+ public void lock() {
+ logger.trace("Obtaining Lock {}", name);
+ lock.lock();
+ lockTime = System.nanoTime();
+ logger.trace("Obtained Lock {}", name);
+ }
+
+ /**
+ * @param task to release the lock for
+ */
+ @Override
+ public void unlock(final String task) {
+ if (lockTime <= 0L) {
+ lock.unlock();
+ return;
+ }
+
+ logger.trace("Releasing Lock {}", name);
+ final long nanosLocked = System.nanoTime() - lockTime;
+
+ Long startIterations = lockIterations.get(task);
+ if (startIterations == null) {
+ startIterations = 0L;
+ }
+ final long iterations = startIterations + 1L;
+ lockIterations.put(task, iterations);
+
+ Long startNanos = lockNanos.get(task);
+ if (startNanos == null) {
+ startNanos = 0L;
+ }
+ final long totalNanos = startNanos + nanosLocked;
+ lockNanos.put(task, totalNanos);
+
+ lockTime = -1L;
+
+ lock.unlock();
+ logger.trace("Released Lock {}", name);
+
+ if (iterations % iterationFrequency == 0) {
+ logger.debug("Lock {} held for {} nanos for task: {}; total lock iterations: {}; total lock nanos: {}", name, nanosLocked, task, iterations, totalNanos);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/concurrency/DebuggableTimedLock.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/concurrency/DebuggableTimedLock.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/concurrency/DebuggableTimedLock.java
new file mode 100644
index 0000000..69da6e8
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/concurrency/DebuggableTimedLock.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.concurrency;
+
+import java.util.concurrent.TimeUnit;
+
+public interface DebuggableTimedLock {
+
+ void lock();
+
+ boolean tryLock(long timePeriod, TimeUnit timeUnit);
+
+ boolean tryLock();
+
+ void unlock(String task);
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/concurrency/TimedLock.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/concurrency/TimedLock.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/concurrency/TimedLock.java
new file mode 100644
index 0000000..532d3c3
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/concurrency/TimedLock.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.concurrency;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TimedLock {
+
+ private final DebugEnabledTimedLock enabled;
+ private final DebugDisabledTimedLock disabled;
+
+ private final Logger logger;
+
+ public TimedLock(final Lock lock, final String name, final int iterationFrequency) {
+ this.enabled = new DebugEnabledTimedLock(lock, name, iterationFrequency);
+ this.disabled = new DebugDisabledTimedLock(lock);
+
+ logger = LoggerFactory.getLogger(TimedLock.class.getName() + "." + name);
+ }
+
+ private DebuggableTimedLock getLock() {
+ return logger.isDebugEnabled() ? enabled : disabled;
+ }
+
+ public boolean tryLock() {
+ return getLock().tryLock();
+ }
+
+ public boolean tryLock(final long timeout, final TimeUnit timeUnit) {
+ return getLock().tryLock(timeout, timeUnit);
+ }
+
+ public void lock() {
+ getLock().lock();
+ }
+
+ public void unlock(final String task) {
+ getLock().unlock(task);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/FileUtils.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/FileUtils.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/FileUtils.java
new file mode 100644
index 0000000..ff4da8e
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/FileUtils.java
@@ -0,0 +1,594 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.file;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Random;
+
+import org.slf4j.Logger;
+
+/**
+ * A utility class containing a few useful static methods to do typical IO operations.
+ *
+ */
+public class FileUtils {
+
+ public static final long TRANSFER_CHUNK_SIZE_BYTES = 1024 * 1024 * 8; //8 MB chunks
+ public static final long MILLIS_BETWEEN_ATTEMPTS = 50L;
+
+ /**
+ * Closes the given closeable quietly - no logging, no exceptions...
+ *
+ * @param closeable the thing to close
+ */
+ public static void closeQuietly(final Closeable closeable) {
+ if (null != closeable) {
+ try {
+ closeable.close();
+ } catch (final IOException io) {/*IGNORE*/
+
+ }
+ }
+ }
+
+ /**
+ * Releases the given lock quietly no logging, no exception
+ *
+ * @param lock the lock to release
+ */
+ public static void releaseQuietly(final FileLock lock) {
+ if (null != lock) {
+ try {
+ lock.release();
+ } catch (final IOException io) {
+ /*IGNORE*/
+ }
+ }
+ }
+
+ public static void ensureDirectoryExistAndCanAccess(final File dir) throws IOException {
+ if (dir.exists() && !dir.isDirectory()) {
+ throw new IOException(dir.getAbsolutePath() + " is not a directory");
+ } else if (!dir.exists()) {
+ final boolean made = dir.mkdirs();
+ if (!made) {
+ throw new IOException(dir.getAbsolutePath() + " could not be created");
+ }
+ }
+ if (!(dir.canRead() && dir.canWrite())) {
+ throw new IOException(dir.getAbsolutePath() + " directory does not have read/write privilege");
+ }
+ }
+
+ /**
+ * Deletes the given file. If the given file exists but could not be deleted this will be printed as a warning to the given logger
+ *
+ * @param file the file to delete
+ * @param logger the logger to provide logging information to about the operation
+ * @return true if given file no longer exists
+ */
+ public static boolean deleteFile(final File file, final Logger logger) {
+ return FileUtils.deleteFile(file, logger, 1);
+ }
+
+ /**
+ * Deletes the given file. If the given file exists but could not be deleted this will be printed as a warning to the given logger
+ *
+ * @param file the file to delete
+ * @param logger the logger to write to
+ * @param attempts indicates how many times an attempt to delete should be made
+ * @return true if given file no longer exists
+ */
+ public static boolean deleteFile(final File file, final Logger logger, final int attempts) {
+ if (file == null) {
+ return false;
+ }
+ boolean isGone = false;
+ try {
+ if (file.exists()) {
+ final int effectiveAttempts = Math.max(1, attempts);
+ for (int i = 0; i < effectiveAttempts && !isGone; i++) {
+ isGone = file.delete() || !file.exists();
+ if (!isGone && (effectiveAttempts - i) > 1) {
+ FileUtils.sleepQuietly(MILLIS_BETWEEN_ATTEMPTS);
+ }
+ }
+ if (!isGone && logger != null) {
+ logger.warn("File appears to exist but unable to delete file: " + file.getAbsolutePath());
+ }
+ }
+ } catch (final Throwable t) {
+ if (logger != null) {
+ logger.warn("Unable to delete file: '" + file.getAbsolutePath() + "' due to " + t);
+ }
+ }
+ return isGone;
+ }
+
+ /**
+ * Deletes all of the given files. If any exist and cannot be deleted that will be printed at warn to the given logger.
+ *
+ * @param files can be null
+ * @param logger can be null
+ */
+ public static void deleteFile(final List<File> files, final Logger logger) {
+ FileUtils.deleteFile(files, logger, 1);
+ }
+
+ /**
+ * Deletes all of the given files. If any exist and cannot be deleted that will be printed at warn to the given logger.
+ *
+ * @param files can be null
+ * @param logger can be null
+ * @param attempts indicates how many times an attempt should be made to delete each file
+ */
+ public static void deleteFile(final List<File> files, final Logger logger, final int attempts) {
+ if (null == files || files.isEmpty()) {
+ return;
+ }
+ final int effectiveAttempts = Math.max(1, attempts);
+ for (final File file : files) {
+ try {
+ boolean isGone = false;
+ for (int i = 0; i < effectiveAttempts && !isGone; i++) {
+ isGone = file.delete() || !file.exists();
+ if (!isGone && (effectiveAttempts - i) > 1) {
+ FileUtils.sleepQuietly(MILLIS_BETWEEN_ATTEMPTS);
+ }
+ }
+ if (!isGone && logger != null) {
+ logger.warn("File appears to exist but unable to delete file: " + file.getAbsolutePath());
+ }
+ } catch (final Throwable t) {
+ if (null != logger) {
+ logger.warn("Unable to delete file given from path: '" + file.getPath() + "' due to " + t);
+ }
+ }
+ }
+ }
+
+ /**
+ * Deletes all files (not directories..) in the given directory (non recursive) that match the given filename filter. If any file cannot be deleted then this is printed at warn to the given
+ * logger.
+ *
+ * @param directory the directory to scan for files to delete
+ * @param filter if null then no filter is used
+ * @param logger the logger to use
+ */
+ public static void deleteFilesInDir(final File directory, final FilenameFilter filter, final Logger logger) {
+ FileUtils.deleteFilesInDir(directory, filter, logger, false);
+ }
+
+ /**
+ * Deletes all files (not directories) in the given directory (recursive) that match the given filename filter. If any file cannot be deleted then this is printed at warn to the given logger.
+ *
+ * @param directory the directory to scan
+ * @param filter if null then no filter is used
+ * @param logger the logger to use
+ * @param recurse indicates whether to recurse subdirectories
+ */
+ public static void deleteFilesInDir(final File directory, final FilenameFilter filter, final Logger logger, final boolean recurse) {
+ FileUtils.deleteFilesInDir(directory, filter, logger, recurse, false);
+ }
+
+ /**
+ * Deletes all files (not directories) in the given directory (recursive) that match the given filename filter. If any file cannot be deleted then this is printed at warn to the given logger.
+ *
+ * @param directory the directory to scan
+ * @param filter if null then no filter is used
+ * @param logger the logger
+ * @param recurse whether to recurse subdirectories or not
+ * @param deleteEmptyDirectories default is false; if true will delete directories found that are empty
+ */
+ public static void deleteFilesInDir(final File directory, final FilenameFilter filter, final Logger logger, final boolean recurse, final boolean deleteEmptyDirectories) {
+ // ensure the specified directory is actually a directory and that it exists
+ if (null != directory && directory.isDirectory()) {
+ final File ingestFiles[] = directory.listFiles();
+ for (File ingestFile : ingestFiles) {
+ boolean process = (filter == null) ? true : filter.accept(directory, ingestFile.getName());
+ if (ingestFile.isFile() && process) {
+ FileUtils.deleteFile(ingestFile, logger, 3);
+ }
+ if (ingestFile.isDirectory() && recurse) {
+ FileUtils.deleteFilesInDir(ingestFile, filter, logger, recurse, deleteEmptyDirectories);
+ if (deleteEmptyDirectories && ingestFile.list().length == 0) {
+ FileUtils.deleteFile(ingestFile, logger, 3);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Deletes given files.
+ *
+ * @param files the files to delete
+ * @param recurse will recurse if true; false otherwise
+ * @throws IOException if any issues deleting specified files
+ */
+ public static void deleteFiles(final Collection<File> files, final boolean recurse) throws IOException {
+ for (final File file : files) {
+ FileUtils.deleteFile(file, recurse);
+ }
+ }
+
+ public static void deleteFile(final File file, final boolean recurse) throws IOException {
+ if (file.isDirectory() && recurse) {
+ FileUtils.deleteFiles(Arrays.asList(file.listFiles()), recurse);
+ }
+ //now delete the file itself regardless of whether it is plain file or a directory
+ if (!FileUtils.deleteFile(file, null, 5)) {
+ throw new IOException("Unable to delete " + file.getAbsolutePath());
+ }
+ }
+
+ /**
+ * Randomly generates a sequence of bytes and overwrites the contents of the file a number of times. The file is then deleted.
+ *
+ * @param file File to be overwritten a number of times and, ultimately, deleted
+ * @param passes Number of times file should be overwritten
+ * @throws IOException if something makes shredding or deleting a problem
+ */
+ public static void shredFile(final File file, final int passes)
+ throws IOException {
+ final Random generator = new Random();
+ final long fileLength = file.length();
+ final int byteArraySize = (int) Math.min(fileLength, 1048576); // 1MB
+ final byte[] b = new byte[byteArraySize];
+ final long numOfRandomWrites = (fileLength / b.length) + 1;
+ final FileOutputStream fos = new FileOutputStream(file);
+ try {
+ // Over write file contents (passes) times
+ final FileChannel channel = fos.getChannel();
+ for (int i = 0; i < passes; i++) {
+ generator.nextBytes(b);
+ for (int j = 0; j <= numOfRandomWrites; j++) {
+ fos.write(b);
+ }
+ fos.flush();
+ channel.position(0);
+ }
+ // Write out "0" for each byte in the file
+ Arrays.fill(b, (byte) 0);
+ for (int j = 0; j < numOfRandomWrites; j++) {
+ fos.write(b);
+ }
+ fos.flush();
+ fos.close();
+ // Try to delete the file a few times
+ if (!FileUtils.deleteFile(file, null, 5)) {
+ throw new IOException("Failed to delete file after shredding");
+ }
+
+ } finally {
+ FileUtils.closeQuietly(fos);
+ }
+ }
+
+ public static long copy(final InputStream in, final OutputStream out) throws IOException {
+ final byte[] buffer = new byte[65536];
+ long copied = 0L;
+ int len;
+ while ((len = in.read(buffer)) > 0) {
+ out.write(buffer, 0, len);
+ copied += len;
+ }
+
+ return copied;
+ }
+
+ public static long copyBytes(final byte[] bytes, final File destination, final boolean lockOutputFile) throws FileNotFoundException, IOException {
+ FileOutputStream fos = null;
+ FileLock outLock = null;
+ long fileSize = 0L;
+ try {
+ fos = new FileOutputStream(destination);
+ final FileChannel out = fos.getChannel();
+ if (lockOutputFile) {
+ outLock = out.tryLock(0, Long.MAX_VALUE, false);
+ if (null == outLock) {
+ throw new IOException("Unable to obtain exclusive file lock for: " + destination.getAbsolutePath());
+ }
+ }
+ fos.write(bytes);
+ fos.flush();
+ fileSize = bytes.length;
+ } finally {
+ FileUtils.releaseQuietly(outLock);
+ FileUtils.closeQuietly(fos);
+ }
+ return fileSize;
+ }
+
+ /**
+ * Copies the given source file to the given destination file. The given destination will be overwritten if it already exists.
+ *
+ * @param source the file to copy
+ * @param destination the file to copy to
+ * @param lockInputFile if true will lock input file during copy; if false will not
+ * @param lockOutputFile if true will lock output file during copy; if false will not
+ * @param move if true will perform what is effectively a move operation rather than a pure copy. This allows for potentially highly efficient movement of the file but if not possible this will
+ * revert to a copy then delete behavior. If false, then the file is copied and the source file is retained. If a true rename/move occurs then no lock is held during that time.
+ * @param logger if failures occur, they will be logged to this logger if possible. If this logger is null, an IOException will instead be thrown, indicating the problem.
+ * @return long number of bytes copied
+ * @throws FileNotFoundException if the source file could not be found
+ * @throws IOException if unable to read or write the underlying streams
+ * @throws SecurityException if a security manager denies the needed file operations
+ */
+ public static long copyFile(final File source, final File destination, final boolean lockInputFile, final boolean lockOutputFile, final boolean move, final Logger logger)
+ throws FileNotFoundException, IOException {
+
+ FileInputStream fis = null;
+ FileOutputStream fos = null;
+ FileLock inLock = null;
+ FileLock outLock = null;
+ long fileSize = 0L;
+ if (!source.canRead()) {
+ throw new IOException("Must at least have read permission");
+
+ }
+ if (move && source.renameTo(destination)) {
+ fileSize = destination.length();
+ } else {
+ try {
+ fis = new FileInputStream(source);
+ fos = new FileOutputStream(destination);
+ final FileChannel in = fis.getChannel();
+ final FileChannel out = fos.getChannel();
+ if (lockInputFile) {
+ inLock = in.tryLock(0, Long.MAX_VALUE, true);
+ if (null == inLock) {
+ throw new IOException("Unable to obtain shared file lock for: " + source.getAbsolutePath());
+ }
+ }
+ if (lockOutputFile) {
+ outLock = out.tryLock(0, Long.MAX_VALUE, false);
+ if (null == outLock) {
+ throw new IOException("Unable to obtain exclusive file lock for: " + destination.getAbsolutePath());
+ }
+ }
+ long bytesWritten = 0;
+ do {
+ bytesWritten += out.transferFrom(in, bytesWritten, TRANSFER_CHUNK_SIZE_BYTES);
+ fileSize = in.size();
+ } while (bytesWritten < fileSize);
+ out.force(false);
+ FileUtils.closeQuietly(fos);
+ FileUtils.closeQuietly(fis);
+ fos = null;
+ fis = null;
+ if (move && !FileUtils.deleteFile(source, null, 5)) {
+ if (logger == null) {
+ FileUtils.deleteFile(destination, null, 5);
+ throw new IOException("Could not remove file " + source.getAbsolutePath());
+ } else {
+ logger.warn("Configured to delete source file when renaming/move not successful. However, unable to delete file at: " + source.getAbsolutePath());
+ }
+ }
+ } finally {
+ FileUtils.releaseQuietly(inLock);
+ FileUtils.releaseQuietly(outLock);
+ FileUtils.closeQuietly(fos);
+ FileUtils.closeQuietly(fis);
+ }
+ }
+ return fileSize;
+ }
+
+ /**
+ * Copies the given source file to the given destination file. The given destination will be overwritten if it already exists.
+ *
+ * @param source the file to copy from
+ * @param destination the file to copy to
+ * @param lockInputFile if true will lock input file during copy; if false will not
+ * @param lockOutputFile if true will lock output file during copy; if false will not
+ * @param logger the logger to use
+ * @return long number of bytes copied
+ * @throws FileNotFoundException if the source file could not be found
+ * @throws IOException if unable to read or write to file
+ * @throws SecurityException if a security manager denies the needed file operations
+ */
+ public static long copyFile(final File source, final File destination, final boolean lockInputFile, final boolean lockOutputFile, final Logger logger) throws FileNotFoundException, IOException {
+ return FileUtils.copyFile(source, destination, lockInputFile, lockOutputFile, false, logger);
+ }
+
+ public static long copyFile(final File source, final OutputStream stream, final boolean closeOutputStream, final boolean lockInputFile) throws FileNotFoundException, IOException {
+ FileInputStream fis = null;
+ FileLock inLock = null;
+ long fileSize = 0L;
+ try {
+ fis = new FileInputStream(source);
+ final FileChannel in = fis.getChannel();
+ if (lockInputFile) {
+ inLock = in.tryLock(0, Long.MAX_VALUE, true);
+ if (inLock == null) {
+ throw new IOException("Unable to obtain exclusive file lock for: " + source.getAbsolutePath());
+ }
+
+ }
+
+ byte[] buffer = new byte[1 << 18]; //256 KB
+ int bytesRead = -1;
+ while ((bytesRead = fis.read(buffer)) != -1) {
+ stream.write(buffer, 0, bytesRead);
+ }
+ in.force(false);
+ stream.flush();
+ fileSize = in.size();
+ } finally {
+ FileUtils.releaseQuietly(inLock);
+ FileUtils.closeQuietly(fis);
+ if (closeOutputStream) {
+ FileUtils.closeQuietly(stream);
+ }
+ }
+ return fileSize;
+ }
+
+ public static long copyFile(final InputStream stream, final File destination, final boolean closeInputStream, final boolean lockOutputFile) throws FileNotFoundException, IOException {
+ final Path destPath = destination.toPath();
+ final long size = Files.copy(stream, destPath);
+ if (closeInputStream) {
+ stream.close();
+ }
+ return size;
+ }
+
+ /**
+ * Renames the given file from the source path to the destination path. This handles multiple attempts. This should only be used to rename within a given directory. Renaming across directories
+ * might not work well. See the <code>File.renameTo</code> for more information.
+ *
+ * @param source the file to rename
+ * @param destination the file path to rename to
+ * @param maxAttempts the max number of attempts to attempt the rename
+ * @throws IOException if rename isn't successful
+ */
+ public static void renameFile(final File source, final File destination, final int maxAttempts) throws IOException {
+ FileUtils.renameFile(source, destination, maxAttempts, false);
+ }
+
+ /**
+ * Renames the given file from the source path to the destination path. This handles multiple attempts. This should only be used to rename within a given directory. Renaming across directories
+ * might not work well. See the <code>File.renameTo</code> for more information.
+ *
+ * @param source the file to rename
+ * @param destination the file path to rename to
+ * @param maxAttempts the max number of attempts to attempt the rename
+ * @param replace if true and a rename attempt fails will check if a file is already at the destination path. If so it will delete that file and attempt the rename according the remaining
+ * maxAttempts. If false, any conflicting files will be left as they were and the rename attempts will fail if conflicting.
+ * @throws IOException if rename isn't successful
+ */
+ public static void renameFile(final File source, final File destination, final int maxAttempts, final boolean replace) throws IOException {
+ final int attempts = (replace || maxAttempts < 1) ? Math.max(2, maxAttempts) : maxAttempts;
+ boolean renamed = false;
+ for (int i = 0; i < attempts; i++) {
+ renamed = source.renameTo(destination);
+ if (!renamed) {
+ FileUtils.deleteFile(destination, null, 5);
+ } else {
+ break; //rename has succeeded
+ }
+ }
+ if (!renamed) {
+ throw new IOException("Attempted " + maxAttempts + " times but unable to rename from \'" + source.getPath() + "\' to \'" + destination.getPath() + "\'");
+
+ }
+ }
+
+ public static void sleepQuietly(final long millis) {
+ try {
+ Thread.sleep(millis);
+ } catch (final InterruptedException ex) {
+ /* do nothing */
+ }
+ }
+
+ /**
+ * Syncs a primary copy of a file with the copy in the restore directory. If the restore directory does not have a file and the primary has a file, the the primary's file is copied to the restore
+ * directory. Else if the restore directory has a file, but the primary does not, then the restore's file is copied to the primary directory. Else if the primary file is different than the restore
+ * file, then an IllegalStateException is thrown. Otherwise, if neither file exists, then no syncing is performed.
+ *
+ * @param primaryFile the primary file
+ * @param restoreFile the restore file
+ * @param logger a logger
+ * @throws IOException if an I/O problem was encountered during syncing
+ * @throws IllegalStateException if the primary and restore copies exist but are different
+ */
+ public static void syncWithRestore(final File primaryFile, final File restoreFile, final Logger logger)
+ throws IOException {
+
+ if (primaryFile.exists() && !restoreFile.exists()) {
+ // copy primary file to restore
+ copyFile(primaryFile, restoreFile, false, false, logger);
+ } else if (restoreFile.exists() && !primaryFile.exists()) {
+ // copy restore file to primary
+ copyFile(restoreFile, primaryFile, false, false, logger);
+ } else if (primaryFile.exists() && restoreFile.exists() && !isSame(primaryFile, restoreFile)) {
+ throw new IllegalStateException(String.format("Primary file '%s' is different than restore file '%s'",
+ primaryFile.getAbsoluteFile(), restoreFile.getAbsolutePath()));
+ }
+ }
+
+ /**
+ * Returns true if the given files are the same according to their MD5 hash.
+ *
+ * @param file1 a file
+ * @param file2 a file
+ * @return true if the files are the same; false otherwise
+ * @throws IOException if the MD5 hash could not be computed
+ */
+ public static boolean isSame(final File file1, final File file2) throws IOException {
+ return Arrays.equals(computeMd5Digest(file1), computeMd5Digest(file2));
+ }
+
+ /**
+ * Returns the MD5 hash of the given file.
+ *
+ * @param file a file
+ * @return the MD5 hash
+ * @throws IOException if the MD5 hash could not be computed
+ */
+ public static byte[] computeMd5Digest(final File file) throws IOException {
+ try (final FileInputStream fis = new FileInputStream(file)) {
+ return computeMd5Digest(fis);
+ }
+ }
+
+ /**
+ * Returns the MD5 hash of the given stream.
+ *
+ * @param stream an input stream
+ * @return the MD5 hash
+ * @throws IOException if the MD5 hash could not be computed
+ */
+ public static byte[] computeMd5Digest(final InputStream stream) throws IOException {
+ final MessageDigest digest;
+ try {
+ digest = MessageDigest.getInstance("MD5");
+ } catch (final NoSuchAlgorithmException nsae) {
+ throw new IOException(nsae);
+ }
+
+
+ int len;
+ final byte[] buffer = new byte[8192];
+ while ((len = stream.read(buffer)) > -1) {
+ if (len > 0) {
+ digest.update(buffer, 0, len);
+ }
+ }
+
+ return digest.digest();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/CompoundUpdateMonitor.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/CompoundUpdateMonitor.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/CompoundUpdateMonitor.java
new file mode 100644
index 0000000..dc60318
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/CompoundUpdateMonitor.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.file.monitor;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * An {@link UpdateMonitor} that combines multiple <code>UpdateMonitor</code>s such that it will indicate a change in a file only if ALL sub-monitors indicate a change. The sub-monitors will be
+ * applied in the order given and if any indicates that the state has not changed, the subsequent sub-monitors may not be given a chance to run
+ */
+public class CompoundUpdateMonitor implements UpdateMonitor {
+
+ private final List<UpdateMonitor> monitors;
+
+ public CompoundUpdateMonitor(final UpdateMonitor first, final UpdateMonitor... others) {
+ monitors = new ArrayList<>();
+ monitors.add(first);
+ for (final UpdateMonitor monitor : others) {
+ monitors.add(monitor);
+ }
+ }
+
+ @Override
+ public Object getCurrentState(final Path path) throws IOException {
+ return new DeferredMonitorAction(monitors, path);
+ }
+
+ private static class DeferredMonitorAction {
+
+ private static final Object NON_COMPUTED_VALUE = new Object();
+
+ private final List<UpdateMonitor> monitors;
+ private final Path path;
+
+ private final Object[] preCalculated;
+
+ public DeferredMonitorAction(final List<UpdateMonitor> monitors, final Path path) {
+ this.monitors = monitors;
+ this.path = path;
+ preCalculated = new Object[monitors.size()];
+
+ for (int i = 0; i < preCalculated.length; i++) {
+ preCalculated[i] = NON_COMPUTED_VALUE;
+ }
+ }
+
+ private Object getCalculatedValue(final int i) throws IOException {
+ if (preCalculated[i] == NON_COMPUTED_VALUE) {
+ preCalculated[i] = monitors.get(i).getCurrentState(path);
+ }
+
+ return preCalculated[i];
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ // must return true unless ALL DeferredMonitorAction's indicate that they are different
+ if (obj == null) {
+ return false;
+ }
+
+ if (!(obj instanceof DeferredMonitorAction)) {
+ return false;
+ }
+
+ final DeferredMonitorAction other = (DeferredMonitorAction) obj;
+ try {
+ // Go through each UpdateMonitor's value and check if the value has changed.
+ for (int i = 0; i < preCalculated.length; i++) {
+ final Object mine = getCalculatedValue(i);
+ final Object theirs = other.getCalculatedValue(i);
+
+ if (mine == theirs) {
+ // same
+ return true;
+ }
+
+ if (mine == null && theirs == null) {
+ // same
+ return true;
+ }
+
+ if (mine.equals(theirs)) {
+ return true;
+ }
+ }
+ } catch (final IOException e) {
+ return false;
+ }
+
+ // No DeferredMonitorAction was the same as last time. Therefore, it's not equal
+ return false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/LastModifiedMonitor.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/LastModifiedMonitor.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/LastModifiedMonitor.java
new file mode 100644
index 0000000..e6be558
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/LastModifiedMonitor.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.file.monitor;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+public class LastModifiedMonitor implements UpdateMonitor {
+
+ @Override
+ public Object getCurrentState(final Path path) throws IOException {
+ return Files.getLastModifiedTime(path);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/MD5SumMonitor.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/MD5SumMonitor.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/MD5SumMonitor.java
new file mode 100644
index 0000000..8dea4bf
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/MD5SumMonitor.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.file.monitor;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Path;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+public class MD5SumMonitor implements UpdateMonitor {
+
+ @Override
+ public Object getCurrentState(final Path path) throws IOException {
+ final MessageDigest digest;
+ try {
+ digest = MessageDigest.getInstance("MD5");
+ } catch (final NoSuchAlgorithmException nsae) {
+ throw new AssertionError(nsae);
+ }
+
+ try (final FileInputStream fis = new FileInputStream(path.toFile())) {
+ int len;
+ final byte[] buffer = new byte[8192];
+ while ((len = fis.read(buffer)) > -1) {
+ if (len > 0) {
+ digest.update(buffer, 0, len);
+ }
+ }
+ }
+
+ // Return a ByteBuffer instead of byte[] because we want equals() to do a deep equality
+ return ByteBuffer.wrap(digest.digest());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/SynchronousFileWatcher.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/SynchronousFileWatcher.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/SynchronousFileWatcher.java
new file mode 100644
index 0000000..0040037
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/SynchronousFileWatcher.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.file.monitor;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * Allows the user to configure a {@link java.nio.file.Path Path} to watch for modifications and periodically poll to check if the file has been modified
+ */
+public class SynchronousFileWatcher {
+
+ private final Path path;
+ private final long checkUpdateMillis;
+ private final UpdateMonitor monitor;
+ private final AtomicReference<StateWrapper> lastState;
+ private final Lock resourceLock = new ReentrantLock();
+
+ public SynchronousFileWatcher(final Path path, final UpdateMonitor monitor) {
+ this(path, monitor, 0L);
+ }
+
+ public SynchronousFileWatcher(final Path path, final UpdateMonitor monitor, final long checkMillis) {
+ if (checkMillis < 0) {
+ throw new IllegalArgumentException();
+ }
+
+ this.path = path;
+ checkUpdateMillis = checkMillis;
+ this.monitor = monitor;
+
+ Object currentState;
+ try {
+ currentState = monitor.getCurrentState(path);
+ } catch (final IOException e) {
+ currentState = null;
+ }
+
+ this.lastState = new AtomicReference<>(new StateWrapper(currentState));
+ }
+
+ /**
+ * Checks if the file has been updated according to the configured {@link UpdateMonitor} and resets the state
+ *
+ * @return true if updated; false otherwise
+ * @throws IOException if failure occurs checking for changes
+ */
+ public boolean checkAndReset() throws IOException {
+ if (checkUpdateMillis <= 0) { // if checkUpdateMillis <= 0, always check
+ return checkForUpdate();
+ } else {
+ final StateWrapper stateWrapper = lastState.get();
+ if (stateWrapper.getTimestamp() < System.currentTimeMillis() - checkUpdateMillis) {
+ return checkForUpdate();
+ }
+ return false;
+ }
+ }
+
+ private boolean checkForUpdate() throws IOException {
+ if (resourceLock.tryLock()) {
+ try {
+ final StateWrapper wrapper = lastState.get();
+ final Object newState = monitor.getCurrentState(path);
+ if (newState == null && wrapper.getState() == null) {
+ return false;
+ }
+ if (newState == null || wrapper.getState() == null) {
+ lastState.set(new StateWrapper(newState));
+ return true;
+ }
+
+ final boolean unmodified = newState.equals(wrapper.getState());
+ if (!unmodified) {
+ lastState.set(new StateWrapper(newState));
+ }
+ return !unmodified;
+ } finally {
+ resourceLock.unlock();
+ }
+ } else {
+ return false;
+ }
+ }
+
+ private static class StateWrapper {
+
+ private final Object state;
+ private final long timestamp;
+
+ public StateWrapper(final Object state) {
+ this.state = state;
+ this.timestamp = System.currentTimeMillis();
+ }
+
+ public Object getState() {
+ return state;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/UpdateMonitor.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/UpdateMonitor.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/UpdateMonitor.java
new file mode 100644
index 0000000..20ed1dd
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/monitor/UpdateMonitor.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.file.monitor;
+
+import java.io.IOException;
+import java.nio.file.Path;
+
+public interface UpdateMonitor {
+
+ Object getCurrentState(Path path) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/search/Search.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/search/Search.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/search/Search.java
new file mode 100644
index 0000000..b407c4d
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/search/Search.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.search;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Set;
+
+import org.apache.nifi.util.search.ahocorasick.SearchState;
+
+/**
+ * Defines an interface to search for content given a set of search terms. Any implementation of search must be thread safe.
+ *
+ */
+public interface Search<T> {
+
+ /**
+ * Establishes the dictionary of terms which will be searched in subsequent search calls. This can be called only once
+ *
+ * @param terms the terms to create a dictionary of
+ */
+ void initializeDictionary(Set<SearchTerm<T>> terms);
+
+ /**
+ * Searches the given input stream for matches between the already specified dictionary and the contents scanned.
+ *
+ * @param haystack the source data to scan for hits
+ * @param findAll if true will find all matches if false will find only the first match
+ * @return SearchState containing results Map might be empty which indicates no matches found but will not be null
+ * @throws IOException Thrown for any exceptions occurring while searching.
+ * @throws IllegalStateException if the dictionary has not yet been initialized
+ */
+ SearchState<T> search(InputStream haystack, boolean findAll) throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/search/SearchTerm.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/search/SearchTerm.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/search/SearchTerm.java
new file mode 100644
index 0000000..48f8678
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/search/SearchTerm.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.search;
+
+import java.nio.charset.Charset;
+import java.util.Arrays;
+
+/**
+ * This is an immutable thread safe object representing a search term
+ *
+ */
+public class SearchTerm<T> {
+
+ private final byte[] bytes;
+ private final int hashCode;
+ private final T reference;
+
+ /**
+ * Constructs a SearchTerm. Defensively copies the given byte array
+ *
+ * @param bytes the bytes of the search term
+ * @throws IllegalArgumentException if given bytes are null or 0 length
+ */
+ public SearchTerm(final byte[] bytes) {
+ this(bytes, true, null);
+ }
+
+ /**
+ * Constructs a search term. Optionally performs a defensive copy of the given byte array. If the caller indicates a defensive copy is not necessary then they must not change the given arrays
+ * state any longer
+ *
+ * @param bytes the bytes of the new search term
+ * @param defensiveCopy if true will make a defensive copy; false otherwise
+ * @param reference a holder for an object which can be retrieved when this search term hits
+ */
+ public SearchTerm(final byte[] bytes, final boolean defensiveCopy, final T reference) {
+ if (bytes == null || bytes.length == 0) {
+ throw new IllegalArgumentException();
+ }
+ if (defensiveCopy) {
+ this.bytes = Arrays.copyOf(bytes, bytes.length);
+ } else {
+ this.bytes = bytes;
+ }
+ this.hashCode = Arrays.hashCode(this.bytes);
+ this.reference = reference;
+ }
+
+ public int get(final int index) {
+ return bytes[index] & 0xff;
+ }
+
+ /**
+ * @return size in of search term in bytes
+ */
+ public int size() {
+ return bytes.length;
+ }
+
+ /**
+ * @return reference object for this given search term
+ */
+ public T getReference() {
+ return reference;
+ }
+
+ /**
+ * Determines if the given window starts with the same bytes as this term
+ *
+ * @param window bytes from the haystack being evaluated
+ * @param windowLength The length of the window to consider
+ * @return true if this term starts with the same bytes of the given window
+ */
+ public boolean startsWith(byte[] window, int windowLength) {
+ if (windowLength > window.length) {
+ throw new IndexOutOfBoundsException();
+ }
+ if (bytes.length < windowLength) {
+ return false;
+ }
+ for (int i = 0; i < bytes.length && i < windowLength; i++) {
+ if (bytes[i] != window[i]) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * @return a defensive copy of the internal byte structure
+ */
+ public byte[] getBytes() {
+ return Arrays.copyOf(bytes, bytes.length);
+ }
+
+ @Override
+ public int hashCode() {
+ return hashCode;
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ final SearchTerm other = (SearchTerm) obj;
+ if (this.hashCode != other.hashCode) {
+ return false;
+ }
+ return Arrays.equals(this.bytes, other.bytes);
+ }
+
+ @Override
+ public String toString() {
+ return new String(bytes);
+ }
+
+ public String toString(final Charset charset) {
+ return new String(bytes, charset);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/search/ahocorasick/AhoCorasick.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/search/ahocorasick/AhoCorasick.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/search/ahocorasick/AhoCorasick.java
new file mode 100644
index 0000000..3b8afaf
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/search/ahocorasick/AhoCorasick.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.search.ahocorasick;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.LinkedList;
+import java.util.Queue;
+import java.util.Set;
+
+import org.apache.nifi.util.search.Search;
+import org.apache.nifi.util.search.SearchTerm;
+
+public class AhoCorasick<T> implements Search<T> {
+
+ private Node root = null;
+
+ /**
+ * Constructs a new search object.
+ *
+ * @throws IllegalArgumentException if given terms are null or empty
+ */
+ public AhoCorasick() {
+ }
+
+ @Override
+ public void initializeDictionary(final Set<SearchTerm<T>> terms) {
+ if (root != null) {
+ throw new IllegalStateException();
+ }
+ root = new Node();
+ if (terms == null || terms.isEmpty()) {
+ throw new IllegalArgumentException();
+ }
+ for (final SearchTerm<T> term : terms) {
+ int i = 0;
+ Node nextNode = root;
+ while (true) {
+ nextNode = addMatch(term, i, nextNode);
+ if (nextNode == null) {
+ break; //we're done
+ }
+ i++;
+ }
+ }
+ initialize();
+ }
+
+ private Node addMatch(final SearchTerm<T> term, final int offset, final Node current) {
+ final int index = term.get(offset);
+ boolean atEnd = (offset == (term.size() - 1));
+ if (current.getNeighbor(index) == null) {
+ if (atEnd) {
+ current.setNeighbor(new Node(term), index);
+ return null;
+ }
+ current.setNeighbor(new Node(), index);
+ } else if (atEnd) {
+ current.getNeighbor(index).setMatchingTerm(term);
+ return null;
+ }
+ return current.getNeighbor(index);
+ }
+
+ private void initialize() {
+ //perform bgs to build failure links
+ final Queue<Node> queue = new LinkedList<>();
+ queue.add(root);
+ root.setFailureNode(null);
+ while (!queue.isEmpty()) {
+ final Node current = queue.poll();
+ for (int i = 0; i < 256; i++) {
+ final Node next = current.getNeighbor(i);
+ if (next != null) {
+ //traverse failure to get state
+ Node fail = current.getFailureNode();
+ while ((fail != null) && fail.getNeighbor(i) == null) {
+ fail = fail.getFailureNode();
+ }
+ if (fail != null) {
+ next.setFailureNode(fail.getNeighbor(i));
+ } else {
+ next.setFailureNode(root);
+ }
+ queue.add(next);
+ }
+ }
+ }
+ }
+
+ @Override
+ public SearchState search(final InputStream stream, final boolean findAll) throws IOException {
+ return search(stream, findAll, null);
+ }
+
+ private SearchState search(final InputStream stream, final boolean findAll, final SearchState state) throws IOException {
+ if (root == null) {
+ throw new IllegalStateException();
+ }
+ final SearchState<T> currentState = (state == null) ? new SearchState(root) : state;
+ if (!findAll && currentState.foundMatch()) {
+ throw new IllegalStateException("A match has already been found yet we're being asked to keep searching");
+ }
+ Node current = currentState.getCurrentNode();
+ int currentChar;
+ while ((currentChar = stream.read()) >= 0) {
+ currentState.incrementBytesRead(1L);
+ Node next = current.getNeighbor(currentChar);
+ if (next == null) {
+ next = current.getFailureNode();
+ while ((next != null) && next.getNeighbor(currentChar) == null) {
+ next = next.getFailureNode();
+ }
+ if (next != null) {
+ next = next.getNeighbor(currentChar);
+ } else {
+ next = root;
+ }
+ }
+ if (next == null) {
+ throw new IllegalStateException("tree out of sync");
+ }
+ //Accept condition
+ if (next.hasMatch()) {
+ currentState.addResult(next.getMatchingTerm());
+ }
+ for (Node failNode = next.getFailureNode(); failNode != null; failNode = failNode.getFailureNode()) {
+ if (failNode.hasMatch()) {
+ currentState.addResult(failNode.getMatchingTerm());
+ }
+ }
+ current = next;
+ if (currentState.foundMatch() && !findAll) {
+ break;//give up as soon as we have at least one match
+ }
+ }
+ currentState.setCurrentNode(current);
+ return currentState;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/search/ahocorasick/Node.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/search/ahocorasick/Node.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/search/ahocorasick/Node.java
new file mode 100644
index 0000000..d61ae6f
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/search/ahocorasick/Node.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.search.ahocorasick;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.nifi.util.search.SearchTerm;
+
+/**
+ *
+ */
+public class Node {
+
+ private final Map<Integer, Node> neighborMap;
+ private Node failureNode;
+ private SearchTerm<?> term;
+
+ Node(final SearchTerm<?> term) {
+ this();
+ this.term = term;
+ }
+
+ Node() {
+ neighborMap = new HashMap<>();
+ term = null;
+ }
+
+ void setFailureNode(final Node fail) {
+ failureNode = fail;
+ }
+
+ public Node getFailureNode() {
+ return failureNode;
+ }
+
+ public boolean hasMatch() {
+ return term != null;
+ }
+
+ void setMatchingTerm(final SearchTerm<?> term) {
+ this.term = term;
+ }
+
+ public SearchTerm<?> getMatchingTerm() {
+ return term;
+ }
+
+ public Node getNeighbor(final int index) {
+ return neighborMap.get(index);
+ }
+
+ void setNeighbor(final Node neighbor, final int index) {
+ neighborMap.put(index, neighbor);
+ }
+
+}
[47/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/controller/repository/RepositoryRecordType.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/repository/RepositoryRecordType.java b/nifi-api/src/main/java/org/apache/nifi/controller/repository/RepositoryRecordType.java
new file mode 100644
index 0000000..ff8dc50
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/repository/RepositoryRecordType.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller.repository;
+
+/**
+ *
+ */
+public enum RepositoryRecordType {
+
+ UPDATE, CREATE, DELETE, CONTENTMISSING, SWAP_IN, SWAP_OUT;
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/controller/repository/claim/ContentClaim.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/repository/claim/ContentClaim.java b/nifi-api/src/main/java/org/apache/nifi/controller/repository/claim/ContentClaim.java
new file mode 100644
index 0000000..53cc44f
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/repository/claim/ContentClaim.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller.repository.claim;
+
+/**
+ * <p>
+ * A ContentClaim is a reference to a given flow file's content. Multiple flow
+ * files may reference the same content by both having the same content
+ * claim.</p>
+ *
+ * <p>
+ * Must be thread safe</p>
+ *
+ */
+public interface ContentClaim extends Comparable<ContentClaim> {
+
+ /**
+ * @return the unique identifier for this claim
+ */
+ String getId();
+
+ /**
+ * @return the container identifier in which this claim is held
+ */
+ String getContainer();
+
+ /**
+ * @return the section within a given container the claim is held
+ */
+ String getSection();
+
+ /**
+ * @return Indicates whether or not the Claim is loss-tolerant. If so, we will
+ * attempt to keep the content but will not sacrifice a great deal of
+ * performance to do so
+ */
+ boolean isLossTolerant();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/controller/repository/claim/ContentClaimManager.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/repository/claim/ContentClaimManager.java b/nifi-api/src/main/java/org/apache/nifi/controller/repository/claim/ContentClaimManager.java
new file mode 100644
index 0000000..bffcec3
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/repository/claim/ContentClaimManager.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller.repository.claim;
+
+import java.util.Collection;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Responsible for managing all ContentClaims that are used in the application
+ */
+public interface ContentClaimManager {
+
+ /**
+ * Creates a new Content Claim with the given id, container, section, and
+ * loss tolerance.
+ *
+ * @param id of claim
+ * @param container of claim
+ * @param section of claim
+ * @param lossTolerant of claim
+ * @return new claim
+ */
+ ContentClaim newContentClaim(String container, String section, String id, boolean lossTolerant);
+
+ /**
+ * @param claim to obtain reference count for
+ * @return the number of FlowFiles that hold a claim to a particular piece
+ * of FlowFile content
+ */
+ int getClaimantCount(ContentClaim claim);
+
+ /**
+ * Decreases by 1 the count of how many FlowFiles hold a claim to a
+ * particular piece of FlowFile content and returns the new count
+ *
+ * @param claim to decrement claimants on
+ * @return new claimaint count
+ */
+ int decrementClaimantCount(ContentClaim claim);
+
+ /**
+ * Increases by 1 the count of how many FlowFiles hold a claim to a
+ * particular piece of FlowFile content and returns the new count
+ *
+ * @param claim to increment claims on
+ * @return new claimant count
+ */
+ int incrementClaimantCount(ContentClaim claim);
+
+ /**
+ * Increases by 1 the count of how many FlowFiles hold a claim to a
+ * particular piece of FlowFile content and returns the new count.
+ *
+ * If it is known that the Content Claim whose count is being incremented is
+ * a newly created ContentClaim, this method should be called with a value
+ * of {@code true} as the second argument, as it may allow the manager to
+ * optimize its tasks, knowing that the Content Claim cannot be referenced
+ * by any other component
+ *
+ * @param claim to increment
+ * @param newClaim provides a hint that no other process can have access to this
+ * claim right now
+ * @return new claim count
+ */
+ int incrementClaimantCount(ContentClaim claim, boolean newClaim);
+
+ /**
+ * Indicates that the given ContentClaim can now be destroyed by the
+ * appropriate Content Repository. This should be done only after it is
+ * guaranteed that the FlowFile Repository has been synchronized with its
+ * underlying storage component. This way, we avoid the following sequence
+ * of events:
+ * <ul>
+ * <li>FlowFile Repository is updated to indicate that FlowFile F no longer
+ * depends on ContentClaim C</li>
+ * <li>ContentClaim C is no longer needed and is destroyed</li>
+ * <li>The Operating System crashes or there is a power failure</li>
+ * <li>Upon restart, the FlowFile Repository was not synchronized with its
+ * underlying storage mechanism and as such indicates that FlowFile F needs
+ * ContentClaim C.</li>
+ * <li>Since ContentClaim C has already been destroyed, it is inaccessible,
+ * and FlowFile F's Content is not found, so the FlowFile is removed,
+ * resulting in data loss.</li>
+ * </ul>
+ *
+ * <p>
+ * Using this method of marking the ContentClaim as destructable only when
+ * the FlowFile repository has been synced with the underlying storage
+ * mechanism, we can ensure that on restart, we will not point to this
+ * unneeded claim. As such, it is now safe to destroy the contents.
+ * </p>
+ *
+ * @param claim to mark as now destructable
+ */
+ void markDestructable(ContentClaim claim);
+
+ /**
+ * Drains up to {@code maxElements} Content Claims from the internal queue
+ * of destructable content claims to the given {@code destination} so that
+ * they can be destroyed.
+ *
+ * @param destination to drain to
+ * @param maxElements max items to drain
+ */
+ void drainDestructableClaims(Collection<ContentClaim> destination, int maxElements);
+
+ /**
+ * Drains up to {@code maxElements} Content Claims from the internal queue
+ * of destructable content claims to the given {@code destination} so that
+ * they can be destroyed. If no ContentClaim is ready to be destroyed at
+ * this time, will wait up to the specified amount of time before returning.
+ * If, after the specified amount of time, there is still no ContentClaim
+ * ready to be destroyed, the method will return without having added
+ * anything to the given {@code destination}.
+ *
+ * @param destination to drain to
+ * @param maxElements max items to drain
+ * @param timeout maximum time to wait
+ * @param unit unit of time to wait
+ */
+ void drainDestructableClaims(Collection<ContentClaim> destination, int maxElements, long timeout, TimeUnit unit);
+
+ /**
+ * Clears the manager's memory of any and all ContentClaims that it knows
+ * about
+ */
+ void purge();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/controller/status/ConnectionStatus.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/status/ConnectionStatus.java b/nifi-api/src/main/java/org/apache/nifi/controller/status/ConnectionStatus.java
new file mode 100644
index 0000000..cebb2b4
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/status/ConnectionStatus.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller.status;
+
+/**
+ */
+public class ConnectionStatus implements Cloneable {
+
+ private String id;
+ private String groupId;
+ private String name;
+ private String sourceId;
+ private String sourceName;
+ private String destinationId;
+ private String destinationName;
+ private int inputCount;
+ private long inputBytes;
+ private int queuedCount;
+ private long queuedBytes;
+ private int outputCount;
+ private long outputBytes;
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(final String id) {
+ this.id = id;
+ }
+
+ public String getGroupId() {
+ return groupId;
+ }
+
+ public void setGroupId(final String groupId) {
+ this.groupId = groupId;
+ }
+
+ public int getQueuedCount() {
+ return queuedCount;
+ }
+
+ public void setQueuedCount(final int queuedCount) {
+ this.queuedCount = queuedCount;
+ }
+
+ public long getQueuedBytes() {
+ return queuedBytes;
+ }
+
+ public void setQueuedBytes(final long queuedBytes) {
+ this.queuedBytes = queuedBytes;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getSourceId() {
+ return sourceId;
+ }
+
+ public void setSourceId(String sourceId) {
+ this.sourceId = sourceId;
+ }
+
+ public String getSourceName() {
+ return sourceName;
+ }
+
+ public void setSourceName(String sourceName) {
+ this.sourceName = sourceName;
+ }
+
+ public String getDestinationId() {
+ return destinationId;
+ }
+
+ public void setDestinationId(String destinationId) {
+ this.destinationId = destinationId;
+ }
+
+ public String getDestinationName() {
+ return destinationName;
+ }
+
+ public void setDestinationName(String destinationName) {
+ this.destinationName = destinationName;
+ }
+
+ public long getInputBytes() {
+ return inputBytes;
+ }
+
+ public void setInputBytes(long inputBytes) {
+ this.inputBytes = inputBytes;
+ }
+
+ public int getInputCount() {
+ return inputCount;
+ }
+
+ public void setInputCount(int inputCount) {
+ this.inputCount = inputCount;
+ }
+
+ public long getOutputBytes() {
+ return outputBytes;
+ }
+
+ public void setOutputBytes(long outputBytes) {
+ this.outputBytes = outputBytes;
+ }
+
+ public int getOutputCount() {
+ return outputCount;
+ }
+
+ public void setOutputCount(int outputCount) {
+ this.outputCount = outputCount;
+ }
+
+ @Override
+ public ConnectionStatus clone() {
+ final ConnectionStatus clonedObj = new ConnectionStatus();
+ clonedObj.groupId = groupId;
+ clonedObj.id = id;
+ clonedObj.inputBytes = inputBytes;
+ clonedObj.inputCount = inputCount;
+ clonedObj.name = name;
+ clonedObj.outputBytes = outputBytes;
+ clonedObj.outputCount = outputCount;
+ clonedObj.queuedBytes = queuedBytes;
+ clonedObj.queuedCount = queuedCount;
+ clonedObj.sourceId = sourceId;
+ clonedObj.sourceName = sourceName;
+ clonedObj.destinationId = destinationId;
+ clonedObj.destinationName = destinationName;
+ return clonedObj;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("ConnectionStatus [id=");
+ builder.append(id);
+ builder.append(", groupId=");
+ builder.append(groupId);
+ builder.append(", name=");
+ builder.append(name);
+ builder.append(", sourceId=");
+ builder.append(sourceId);
+ builder.append(", sourceName=");
+ builder.append(sourceName);
+ builder.append(", destinationId=");
+ builder.append(destinationId);
+ builder.append(", destinationName=");
+ builder.append(destinationName);
+ builder.append(", inputCount=");
+ builder.append(inputCount);
+ builder.append(", inputBytes=");
+ builder.append(inputBytes);
+ builder.append(", queuedCount=");
+ builder.append(queuedCount);
+ builder.append(", queuedBytes=");
+ builder.append(queuedBytes);
+ builder.append(", outputCount=");
+ builder.append(outputCount);
+ builder.append(", outputBytes=");
+ builder.append(outputBytes);
+ builder.append("]");
+ return builder.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/controller/status/PortStatus.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/status/PortStatus.java b/nifi-api/src/main/java/org/apache/nifi/controller/status/PortStatus.java
new file mode 100644
index 0000000..0d248cd
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/status/PortStatus.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller.status;
+
+/**
+ * The status of a port.
+ */
+public class PortStatus implements Cloneable {
+
+ private String id;
+ private String groupId;
+ private String name;
+ private Integer activeThreadCount;
+ private int inputCount;
+ private long inputBytes;
+ private int outputCount;
+ private long outputBytes;
+ private int flowFilesReceived;
+ private long bytesReceived;
+ private int flowFilesSent;
+ private long bytesSent;
+ private Boolean transmitting;
+ private RunStatus runStatus;
+
+ public Boolean isTransmitting() {
+ return transmitting;
+ }
+
+ public void setTransmitting(Boolean transmitting) {
+ this.transmitting = transmitting;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ public String getGroupId() {
+ return groupId;
+ }
+
+ public void setGroupId(String groupId) {
+ this.groupId = groupId;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public Integer getActiveThreadCount() {
+ return activeThreadCount;
+ }
+
+ public void setActiveThreadCount(Integer activeThreadCount) {
+ this.activeThreadCount = activeThreadCount;
+ }
+
+ public int getInputCount() {
+ return inputCount;
+ }
+
+ public void setInputCount(int inputCount) {
+ this.inputCount = inputCount;
+ }
+
+ public long getInputBytes() {
+ return inputBytes;
+ }
+
+ public void setInputBytes(long inputBytes) {
+ this.inputBytes = inputBytes;
+ }
+
+ public int getOutputCount() {
+ return outputCount;
+ }
+
+ public void setOutputCount(final int outputCount) {
+ this.outputCount = outputCount;
+ }
+
+ public long getOutputBytes() {
+ return outputBytes;
+ }
+
+ public void setOutputBytes(final long outputBytes) {
+ this.outputBytes = outputBytes;
+ }
+
+ public RunStatus getRunStatus() {
+ return runStatus;
+ }
+
+ public void setRunStatus(RunStatus runStatus) {
+ this.runStatus = runStatus;
+ }
+
+ public int getFlowFilesReceived() {
+ return flowFilesReceived;
+ }
+
+ public void setFlowFilesReceived(int flowFilesReceived) {
+ this.flowFilesReceived = flowFilesReceived;
+ }
+
+ public long getBytesReceived() {
+ return bytesReceived;
+ }
+
+ public void setBytesReceived(long bytesReceived) {
+ this.bytesReceived = bytesReceived;
+ }
+
+ public int getFlowFilesSent() {
+ return flowFilesSent;
+ }
+
+ public void setFlowFilesSent(int flowFilesSent) {
+ this.flowFilesSent = flowFilesSent;
+ }
+
+ public long getBytesSent() {
+ return bytesSent;
+ }
+
+ public void setBytesSent(long bytesSent) {
+ this.bytesSent = bytesSent;
+ }
+
+ public Boolean getTransmitting() {
+ return transmitting;
+ }
+
+ @Override
+ public PortStatus clone() {
+ final PortStatus clonedObj = new PortStatus();
+ clonedObj.id = id;
+ clonedObj.groupId = groupId;
+ clonedObj.name = name;
+ clonedObj.activeThreadCount = activeThreadCount;
+ clonedObj.inputBytes = inputBytes;
+ clonedObj.inputCount = inputCount;
+ clonedObj.outputBytes = outputBytes;
+ clonedObj.outputCount = outputCount;
+ clonedObj.flowFilesReceived = flowFilesReceived;
+ clonedObj.bytesReceived = bytesReceived;
+ clonedObj.flowFilesSent = flowFilesSent;
+ clonedObj.bytesSent = bytesSent;
+ clonedObj.transmitting = transmitting;
+ clonedObj.runStatus = runStatus;
+ return clonedObj;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("PortStatus [id=");
+ builder.append(id);
+ builder.append(", groupId=");
+ builder.append(groupId);
+ builder.append(", name=");
+ builder.append(name);
+ builder.append(", activeThreadCount=");
+ builder.append(activeThreadCount);
+ builder.append(", transmitting=");
+ builder.append(transmitting);
+ builder.append(", inputCount=");
+ builder.append(inputCount);
+ builder.append(", inputBytes=");
+ builder.append(inputBytes);
+ builder.append(", outputCount=");
+ builder.append(outputCount);
+ builder.append(", outputBytes=");
+ builder.append(outputBytes);
+ builder.append(", runStatus=");
+ builder.append(runStatus);
+ builder.append("]");
+ return builder.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/controller/status/ProcessGroupStatus.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/status/ProcessGroupStatus.java b/nifi-api/src/main/java/org/apache/nifi/controller/status/ProcessGroupStatus.java
new file mode 100644
index 0000000..eb0339f
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/status/ProcessGroupStatus.java
@@ -0,0 +1,584 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller.status;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ */
+public class ProcessGroupStatus implements Cloneable {
+
+ private String id;
+ private String name;
+ private Integer inputCount;
+ private Long inputContentSize;
+ private Integer outputCount;
+ private Long outputContentSize;
+ private long creationTimestamp;
+ private Integer activeThreadCount;
+ private Integer queuedCount;
+ private Long queuedContentSize;
+ private Long bytesRead;
+ private Long bytesWritten;
+ private int flowFilesReceived;
+ private long bytesReceived;
+ private int flowFilesSent;
+ private long bytesSent;
+ private int flowFilesTransferred;
+ private long bytesTransferred;
+
+ private Collection<ConnectionStatus> connectionStatus = new ArrayList<>();
+ private Collection<ProcessorStatus> processorStatus = new ArrayList<>();
+ private Collection<ProcessGroupStatus> processGroupStatus = new ArrayList<>();
+ private Collection<RemoteProcessGroupStatus> remoteProcessGroupStatus = new ArrayList<>();
+ private Collection<PortStatus> inputPortStatus = new ArrayList<>();
+ private Collection<PortStatus> outputPortStatus = new ArrayList<>();
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(final String id) {
+ this.id = id;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public Integer getInputCount() {
+ return inputCount;
+ }
+
+ public void setInputCount(final Integer inputCount) {
+ this.inputCount = inputCount;
+ }
+
+ public Long getInputContentSize() {
+ return inputContentSize;
+ }
+
+ public void setInputContentSize(final Long inputContentSize) {
+ this.inputContentSize = inputContentSize;
+ }
+
+ public Integer getOutputCount() {
+ return outputCount;
+ }
+
+ public void setOutputCount(final Integer outputCount) {
+ this.outputCount = outputCount;
+ }
+
+ public Long getOutputContentSize() {
+ return outputContentSize;
+ }
+
+ public void setOutputContentSize(final Long outputContentSize) {
+ this.outputContentSize = outputContentSize;
+ }
+
+ public Long getBytesRead() {
+ return bytesRead;
+ }
+
+ public void setBytesRead(final Long bytesRead) {
+ this.bytesRead = bytesRead;
+ }
+
+ public Long getBytesWritten() {
+ return bytesWritten;
+ }
+
+ public void setBytesWritten(final Long bytesWritten) {
+ this.bytesWritten = bytesWritten;
+ }
+
+ public Integer getQueuedCount() {
+ return this.queuedCount;
+ }
+
+ public void setQueuedCount(final Integer queuedCount) {
+ this.queuedCount = queuedCount;
+ }
+
+ public Long getQueuedContentSize() {
+ return queuedContentSize;
+ }
+
+ public void setQueuedContentSize(final Long queuedContentSize) {
+ this.queuedContentSize = queuedContentSize;
+ }
+
+ public long getCreationTimestamp() {
+ return creationTimestamp;
+ }
+
+ public void setCreationTimestamp(final long creationTimestamp) {
+ this.creationTimestamp = creationTimestamp;
+ }
+
+ public Integer getActiveThreadCount() {
+ return activeThreadCount;
+ }
+
+ public void setActiveThreadCount(final Integer activeThreadCount) {
+ this.activeThreadCount = activeThreadCount;
+ }
+
+ public Collection<ConnectionStatus> getConnectionStatus() {
+ return connectionStatus;
+ }
+
+ public void setConnectionStatus(final Collection<ConnectionStatus> connectionStatus) {
+ this.connectionStatus = connectionStatus;
+ }
+
+ public Collection<ProcessorStatus> getProcessorStatus() {
+ return processorStatus;
+ }
+
+ public void setProcessorStatus(final Collection<ProcessorStatus> processorStatus) {
+ this.processorStatus = processorStatus;
+ }
+
+ public Collection<ProcessGroupStatus> getProcessGroupStatus() {
+ return processGroupStatus;
+ }
+
+ public void setProcessGroupStatus(final Collection<ProcessGroupStatus> processGroupStatus) {
+ this.processGroupStatus = processGroupStatus;
+ }
+
+ public Collection<PortStatus> getInputPortStatus() {
+ return inputPortStatus;
+ }
+
+ public void setInputPortStatus(Collection<PortStatus> inputPortStatus) {
+ this.inputPortStatus = inputPortStatus;
+ }
+
+ public Collection<PortStatus> getOutputPortStatus() {
+ return outputPortStatus;
+ }
+
+ public void setOutputPortStatus(Collection<PortStatus> outputPortStatus) {
+ this.outputPortStatus = outputPortStatus;
+ }
+
+ public Collection<RemoteProcessGroupStatus> getRemoteProcessGroupStatus() {
+ return remoteProcessGroupStatus;
+ }
+
+ public void setRemoteProcessGroupStatus(final Collection<RemoteProcessGroupStatus> remoteProcessGroupStatus) {
+ this.remoteProcessGroupStatus = remoteProcessGroupStatus;
+ }
+
+ public int getFlowFilesReceived() {
+ return flowFilesReceived;
+ }
+
+ public void setFlowFilesReceived(final int flowFilesReceived) {
+ this.flowFilesReceived = flowFilesReceived;
+ }
+
+ public long getBytesReceived() {
+ return bytesReceived;
+ }
+
+ public void setBytesReceived(final long bytesReceived) {
+ this.bytesReceived = bytesReceived;
+ }
+
+ public int getFlowFilesSent() {
+ return flowFilesSent;
+ }
+
+ public void setFlowFilesSent(final int flowFilesSent) {
+ this.flowFilesSent = flowFilesSent;
+ }
+
+ public long getBytesSent() {
+ return bytesSent;
+ }
+
+ public void setBytesSent(final long bytesSent) {
+ this.bytesSent = bytesSent;
+ }
+
+ public int getFlowFilesTransferred() {
+ return flowFilesTransferred;
+ }
+
+ public void setFlowFilesTransferred(int flowFilesTransferred) {
+ this.flowFilesTransferred = flowFilesTransferred;
+ }
+
+ public long getBytesTransferred() {
+ return bytesTransferred;
+ }
+
+ public void setBytesTransferred(long bytesTransferred) {
+ this.bytesTransferred = bytesTransferred;
+ }
+
+ @Override
+ public ProcessGroupStatus clone() {
+
+ final ProcessGroupStatus clonedObj = new ProcessGroupStatus();
+
+ clonedObj.creationTimestamp = creationTimestamp;
+ clonedObj.id = id;
+ clonedObj.name = name;
+ clonedObj.outputContentSize = outputContentSize;
+ clonedObj.outputCount = outputCount;
+ clonedObj.inputContentSize = inputContentSize;
+ clonedObj.inputCount = inputCount;
+ clonedObj.activeThreadCount = activeThreadCount;
+ clonedObj.queuedContentSize = queuedContentSize;
+ clonedObj.queuedCount = queuedCount;
+ clonedObj.bytesRead = bytesRead;
+ clonedObj.bytesWritten = bytesWritten;
+ clonedObj.flowFilesReceived = flowFilesReceived;
+ clonedObj.bytesReceived = bytesReceived;
+ clonedObj.flowFilesSent = flowFilesSent;
+ clonedObj.bytesSent = bytesSent;
+ clonedObj.flowFilesTransferred = flowFilesTransferred;
+ clonedObj.bytesTransferred = bytesTransferred;
+
+ if (connectionStatus != null) {
+ final Collection<ConnectionStatus> statusList = new ArrayList<>();
+ clonedObj.setConnectionStatus(statusList);
+ for (final ConnectionStatus status : connectionStatus) {
+ statusList.add(status.clone());
+ }
+ }
+
+ if (processorStatus != null) {
+ final Collection<ProcessorStatus> statusList = new ArrayList<>();
+ clonedObj.setProcessorStatus(statusList);
+ for (final ProcessorStatus status : processorStatus) {
+ statusList.add(status.clone());
+ }
+ }
+
+ if (inputPortStatus != null) {
+ final Collection<PortStatus> statusList = new ArrayList<>();
+ clonedObj.setInputPortStatus(statusList);
+ for (final PortStatus status : inputPortStatus) {
+ statusList.add(status.clone());
+ }
+ }
+
+ if (outputPortStatus != null) {
+ final Collection<PortStatus> statusList = new ArrayList<>();
+ clonedObj.setOutputPortStatus(statusList);
+ for (final PortStatus status : outputPortStatus) {
+ statusList.add(status.clone());
+ }
+ }
+
+ if (processGroupStatus != null) {
+ final Collection<ProcessGroupStatus> statusList = new ArrayList<>();
+ clonedObj.setProcessGroupStatus(statusList);
+ for (final ProcessGroupStatus status : processGroupStatus) {
+ statusList.add(status.clone());
+ }
+ }
+
+ if (remoteProcessGroupStatus != null) {
+ final Collection<RemoteProcessGroupStatus> statusList = new ArrayList<>();
+ clonedObj.setRemoteProcessGroupStatus(statusList);
+ for (final RemoteProcessGroupStatus status : remoteProcessGroupStatus) {
+ statusList.add(status.clone());
+ }
+ }
+
+ return clonedObj;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("ProcessGroupStatus [id=");
+ builder.append(id);
+ builder.append(", inputCount=");
+ builder.append(inputCount);
+ builder.append(", inputBytes=");
+ builder.append(inputContentSize);
+ builder.append(", outputCount=");
+ builder.append(outputCount);
+ builder.append(", outputBytes=");
+ builder.append(outputContentSize);
+ builder.append(", creationTimestamp=");
+ builder.append(creationTimestamp);
+ builder.append(", activeThreadCount=");
+ builder.append(activeThreadCount);
+ builder.append(", flowFilesTransferred=");
+ builder.append(flowFilesTransferred);
+ builder.append(", bytesTransferred=");
+ builder.append(bytesTransferred);
+ builder.append(", flowFilesReceived=");
+ builder.append(flowFilesReceived);
+ builder.append(", bytesReceived=");
+ builder.append(bytesReceived);
+ builder.append(", flowFilesSent=");
+ builder.append(flowFilesSent);
+ builder.append(", bytesSent=");
+ builder.append(bytesSent);
+ builder.append(",\n\tconnectionStatus=");
+
+ for (final ConnectionStatus status : connectionStatus) {
+ builder.append("\n\t\t");
+ builder.append(status);
+ }
+
+ builder.append(",\n\tprocessorStatus=");
+
+ for (final ProcessorStatus status : processorStatus) {
+ builder.append("\n\t\t");
+ builder.append(status);
+ }
+
+ builder.append(",\n\tprocessGroupStatus=");
+
+ for (final ProcessGroupStatus status : processGroupStatus) {
+ builder.append("\n\t\t");
+ builder.append(status);
+ }
+
+ builder.append(",\n\tremoteProcessGroupStatus=");
+ for (final RemoteProcessGroupStatus status : remoteProcessGroupStatus) {
+ builder.append("\n\t\t");
+ builder.append(status);
+ }
+
+ builder.append(",\n\tinputPortStatus=");
+ for (final PortStatus status : inputPortStatus) {
+ builder.append("\n\t\t");
+ builder.append(status);
+ }
+
+ builder.append(",\n\toutputPortStatus=");
+ for (final PortStatus status : outputPortStatus) {
+ builder.append("\n\t\t");
+ builder.append(status);
+ }
+
+ builder.append("]");
+ return builder.toString();
+ }
+
+ public static void merge(final ProcessGroupStatus target, final ProcessGroupStatus toMerge) {
+ if (target == null || toMerge == null) {
+ return;
+ }
+
+ target.setInputCount(target.getInputCount() + toMerge.getInputCount());
+ target.setInputContentSize(target.getInputContentSize() + toMerge.getInputContentSize());
+ target.setOutputCount(target.getOutputCount() + toMerge.getOutputCount());
+ target.setOutputContentSize(target.getOutputContentSize() + toMerge.getOutputContentSize());
+ target.setQueuedCount(target.getQueuedCount() + toMerge.getQueuedCount());
+ target.setQueuedContentSize(target.getQueuedContentSize() + toMerge.getQueuedContentSize());
+ target.setBytesRead(target.getBytesRead() + toMerge.getBytesRead());
+ target.setBytesWritten(target.getBytesWritten() + toMerge.getBytesWritten());
+ target.setActiveThreadCount(target.getActiveThreadCount() + toMerge.getActiveThreadCount());
+ target.setFlowFilesTransferred(target.getFlowFilesTransferred() + toMerge.getFlowFilesTransferred());
+ target.setBytesTransferred(target.getBytesTransferred() + toMerge.getBytesTransferred());
+ target.setFlowFilesReceived(target.getFlowFilesReceived() + toMerge.getFlowFilesReceived());
+ target.setBytesReceived(target.getBytesReceived() + toMerge.getBytesReceived());
+ target.setFlowFilesSent(target.getFlowFilesSent() + toMerge.getFlowFilesSent());
+ target.setBytesSent(target.getBytesSent() + toMerge.getBytesSent());
+
+ // connection status
+ // sort by id
+ final Map<String, ConnectionStatus> mergedConnectionMap = new HashMap<>();
+ for (final ConnectionStatus status : target.getConnectionStatus()) {
+ mergedConnectionMap.put(status.getId(), status);
+ }
+
+ for (final ConnectionStatus statusToMerge : toMerge.getConnectionStatus()) {
+ ConnectionStatus merged = mergedConnectionMap.get(statusToMerge.getId());
+ if (merged == null) {
+ mergedConnectionMap.put(statusToMerge.getId(), statusToMerge.clone());
+ continue;
+ }
+
+ merged.setQueuedCount(merged.getQueuedCount() + statusToMerge.getQueuedCount());
+ merged.setQueuedBytes(merged.getQueuedBytes() + statusToMerge.getQueuedBytes());
+ merged.setInputCount(merged.getInputCount() + statusToMerge.getInputCount());
+ merged.setInputBytes(merged.getInputBytes() + statusToMerge.getInputBytes());
+ merged.setOutputCount(merged.getOutputCount() + statusToMerge.getOutputCount());
+ merged.setOutputBytes(merged.getOutputBytes() + statusToMerge.getOutputBytes());
+ }
+ target.setConnectionStatus(mergedConnectionMap.values());
+
+ // processor status
+ final Map<String, ProcessorStatus> mergedProcessorMap = new HashMap<>();
+ for (final ProcessorStatus status : target.getProcessorStatus()) {
+ mergedProcessorMap.put(status.getId(), status);
+ }
+
+ for (final ProcessorStatus statusToMerge : toMerge.getProcessorStatus()) {
+ ProcessorStatus merged = mergedProcessorMap.get(statusToMerge.getId());
+ if (merged == null) {
+ mergedProcessorMap.put(statusToMerge.getId(), statusToMerge.clone());
+ continue;
+ }
+
+ merged.setActiveThreadCount(merged.getActiveThreadCount() + statusToMerge.getActiveThreadCount());
+ merged.setBytesRead(merged.getBytesRead() + statusToMerge.getBytesRead());
+ merged.setBytesWritten(merged.getBytesWritten() + statusToMerge.getBytesWritten());
+ merged.setInputBytes(merged.getInputBytes() + statusToMerge.getInputBytes());
+ merged.setInputCount(merged.getInputCount() + statusToMerge.getInputCount());
+ merged.setInvocations(merged.getInvocations() + statusToMerge.getInvocations());
+ merged.setOutputBytes(merged.getOutputBytes() + statusToMerge.getOutputBytes());
+ merged.setOutputCount(merged.getOutputCount() + statusToMerge.getOutputCount());
+ merged.setProcessingNanos(merged.getProcessingNanos() + statusToMerge.getProcessingNanos());
+
+ // if the status to merge is invalid allow it to take precedence. whether the
+ // processor run status is disabled/stopped/running is part of the flow configuration
+ // and should not differ amongst nodes. however, whether a processor is invalid
+ // can be driven by environmental conditions. this check allows any of those to
+ // take precedence over the configured run status.
+ if (RunStatus.Invalid.equals(statusToMerge.getRunStatus())) {
+ merged.setRunStatus(RunStatus.Invalid);
+ }
+ }
+ target.setProcessorStatus(mergedProcessorMap.values());
+
+ // input ports
+ final Map<String, PortStatus> mergedInputPortMap = new HashMap<>();
+ for (final PortStatus status : target.getInputPortStatus()) {
+ mergedInputPortMap.put(status.getId(), status);
+ }
+
+ for (final PortStatus statusToMerge : toMerge.getInputPortStatus()) {
+ PortStatus merged = mergedInputPortMap.get(statusToMerge.getId());
+ if (merged == null) {
+ mergedInputPortMap.put(statusToMerge.getId(), statusToMerge.clone());
+ continue;
+ }
+
+ merged.setInputBytes(merged.getInputBytes() + statusToMerge.getInputBytes());
+ merged.setInputCount(merged.getInputCount() + statusToMerge.getInputCount());
+ merged.setOutputBytes(merged.getOutputBytes() + statusToMerge.getOutputBytes());
+ merged.setOutputCount(merged.getOutputCount() + statusToMerge.getOutputCount());
+ merged.setActiveThreadCount(merged.getActiveThreadCount() + statusToMerge.getActiveThreadCount());
+ if (statusToMerge.isTransmitting() != null && statusToMerge.isTransmitting()) {
+ merged.setTransmitting(true);
+ }
+
+ // should be unnecessary here since ports run status should not be affected by
+ // environmental conditions but doing so in case that changes
+ if (RunStatus.Invalid.equals(statusToMerge.getRunStatus())) {
+ merged.setRunStatus(RunStatus.Invalid);
+ }
+ }
+ target.setInputPortStatus(mergedInputPortMap.values());
+
+ // output ports
+ final Map<String, PortStatus> mergedOutputPortMap = new HashMap<>();
+ for (final PortStatus status : target.getOutputPortStatus()) {
+ mergedOutputPortMap.put(status.getId(), status);
+ }
+
+ for (final PortStatus statusToMerge : toMerge.getOutputPortStatus()) {
+ PortStatus merged = mergedOutputPortMap.get(statusToMerge.getId());
+ if (merged == null) {
+ mergedOutputPortMap.put(statusToMerge.getId(), statusToMerge.clone());
+ continue;
+ }
+
+ merged.setInputBytes(merged.getInputBytes() + statusToMerge.getInputBytes());
+ merged.setInputCount(merged.getInputCount() + statusToMerge.getInputCount());
+ merged.setOutputBytes(merged.getOutputBytes() + statusToMerge.getOutputBytes());
+ merged.setOutputCount(merged.getOutputCount() + statusToMerge.getOutputCount());
+ merged.setActiveThreadCount(merged.getActiveThreadCount() + statusToMerge.getActiveThreadCount());
+ if (statusToMerge.isTransmitting() != null && statusToMerge.isTransmitting()) {
+ merged.setTransmitting(true);
+ }
+
+ // should be unnecessary here since ports run status not should be affected by
+ // environmental conditions but doing so in case that changes
+ if (RunStatus.Invalid.equals(statusToMerge.getRunStatus())) {
+ merged.setRunStatus(RunStatus.Invalid);
+ }
+ }
+ target.setOutputPortStatus(mergedOutputPortMap.values());
+
+ // child groups
+ final Map<String, ProcessGroupStatus> mergedGroupMap = new HashMap<>();
+ for (final ProcessGroupStatus status : target.getProcessGroupStatus()) {
+ mergedGroupMap.put(status.getId(), status);
+ }
+
+ for (final ProcessGroupStatus statusToMerge : toMerge.getProcessGroupStatus()) {
+ ProcessGroupStatus merged = mergedGroupMap.get(statusToMerge.getId());
+ if (merged == null) {
+ mergedGroupMap.put(statusToMerge.getId(), statusToMerge.clone());
+ continue;
+ }
+
+ merge(merged, statusToMerge);
+ }
+ target.setOutputPortStatus(mergedOutputPortMap.values());
+
+ // remote groups
+ final Map<String, RemoteProcessGroupStatus> mergedRemoteGroupMap = new HashMap<>();
+ for (final RemoteProcessGroupStatus status : target.getRemoteProcessGroupStatus()) {
+ mergedRemoteGroupMap.put(status.getId(), status);
+ }
+
+ for (final RemoteProcessGroupStatus statusToMerge : toMerge.getRemoteProcessGroupStatus()) {
+ RemoteProcessGroupStatus merged = mergedRemoteGroupMap.get(statusToMerge.getId());
+ if (merged == null) {
+ mergedRemoteGroupMap.put(statusToMerge.getId(), statusToMerge.clone());
+ continue;
+ }
+
+ // NOTE - active/inactive port counts are not merged since that state is considered part of the flow (like runStatus)
+ merged.setReceivedContentSize(merged.getReceivedContentSize() + statusToMerge.getReceivedContentSize());
+ merged.setReceivedCount(merged.getReceivedCount() + statusToMerge.getReceivedCount());
+ merged.setSentContentSize(merged.getSentContentSize() + statusToMerge.getSentContentSize());
+ merged.setSentCount(merged.getSentCount() + statusToMerge.getSentCount());
+ merged.setActiveThreadCount(merged.getActiveThreadCount() + statusToMerge.getActiveThreadCount());
+
+ List<String> mergedAuthenticationIssues = merged.getAuthorizationIssues();
+ if (mergedAuthenticationIssues == null) {
+ mergedAuthenticationIssues = new ArrayList<>();
+ }
+
+ final List<String> nodeAuthorizationIssues = statusToMerge.getAuthorizationIssues();
+ if (nodeAuthorizationIssues != null && !nodeAuthorizationIssues.isEmpty()) {
+ mergedAuthenticationIssues.addAll(nodeAuthorizationIssues);
+ }
+
+ merged.setAuthorizationIssues(mergedAuthenticationIssues);
+ }
+
+ target.setRemoteProcessGroupStatus(mergedRemoteGroupMap.values());
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/controller/status/ProcessorStatus.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/status/ProcessorStatus.java b/nifi-api/src/main/java/org/apache/nifi/controller/status/ProcessorStatus.java
new file mode 100644
index 0000000..54be7ba
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/status/ProcessorStatus.java
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller.status;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ */
+public class ProcessorStatus implements Cloneable {
+
+ private String id;
+ private String groupId;
+ private String name;
+ private String type;
+ private RunStatus runStatus;
+ private int inputCount;
+ private long inputBytes;
+ private int outputCount;
+ private long outputBytes;
+ private long bytesRead;
+ private long bytesWritten;
+ private int invocations;
+ private long processingNanos;
+ private int flowFilesRemoved;
+ private long averageLineageDuration;
+ private int activeThreadCount;
+ private int flowFilesReceived;
+ private long bytesReceived;
+ private int flowFilesSent;
+ private long bytesSent;
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(final String id) {
+ this.id = id;
+ }
+
+ public String getGroupId() {
+ return groupId;
+ }
+
+ public void setGroupId(final String groupId) {
+ this.groupId = groupId;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(final String name) {
+ this.name = name;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(final String type) {
+ this.type = type;
+ }
+
+ public int getInputCount() {
+ return inputCount;
+ }
+
+ public RunStatus getRunStatus() {
+ return runStatus;
+ }
+
+ public void setRunStatus(RunStatus runStatus) {
+ this.runStatus = runStatus;
+ }
+
+ public void setInputCount(final int inputCount) {
+ this.inputCount = inputCount;
+ }
+
+ public long getInputBytes() {
+ return inputBytes;
+ }
+
+ public void setInputBytes(final long inputBytes) {
+ this.inputBytes = inputBytes;
+ }
+
+ public int getOutputCount() {
+ return outputCount;
+ }
+
+ public void setOutputCount(final int outputCount) {
+ this.outputCount = outputCount;
+ }
+
+ public long getOutputBytes() {
+ return outputBytes;
+ }
+
+ public void setOutputBytes(final long outputBytes) {
+ this.outputBytes = outputBytes;
+ }
+
+ public long getBytesRead() {
+ return bytesRead;
+ }
+
+ public void setBytesRead(final long bytesRead) {
+ this.bytesRead = bytesRead;
+ }
+
+ public long getBytesWritten() {
+ return bytesWritten;
+ }
+
+ public void setBytesWritten(final long bytesWritten) {
+ this.bytesWritten = bytesWritten;
+ }
+
+ public int getInvocations() {
+ return invocations;
+ }
+
+ public void setInvocations(final int invocations) {
+ this.invocations = invocations;
+ }
+
+ public long getProcessingNanos() {
+ return processingNanos;
+ }
+
+ public void setProcessingNanos(final long processingNanos) {
+ this.processingNanos = processingNanos;
+ }
+
+ public long getAverageLineageDuration(final TimeUnit timeUnit) {
+ return TimeUnit.MILLISECONDS.convert(averageLineageDuration, timeUnit);
+ }
+
+ public void setAverageLineageDuration(final long duration, final TimeUnit timeUnit) {
+ this.averageLineageDuration = timeUnit.toMillis(duration);
+ }
+
+ public long getAverageLineageDuration() {
+ return averageLineageDuration;
+ }
+
+ public void setAverageLineageDuration(final long millis) {
+ this.averageLineageDuration = millis;
+ }
+
+ public int getFlowFilesRemoved() {
+ return flowFilesRemoved;
+ }
+
+ public void setFlowFilesRemoved(int flowFilesRemoved) {
+ this.flowFilesRemoved = flowFilesRemoved;
+ }
+
+ public int getActiveThreadCount() {
+ return activeThreadCount;
+ }
+
+ public void setActiveThreadCount(final int activeThreadCount) {
+ this.activeThreadCount = activeThreadCount;
+ }
+
+ public int getFlowFilesReceived() {
+ return flowFilesReceived;
+ }
+
+ public void setFlowFilesReceived(int flowFilesReceived) {
+ this.flowFilesReceived = flowFilesReceived;
+ }
+
+ public long getBytesReceived() {
+ return bytesReceived;
+ }
+
+ public void setBytesReceived(long bytesReceived) {
+ this.bytesReceived = bytesReceived;
+ }
+
+ public int getFlowFilesSent() {
+ return flowFilesSent;
+ }
+
+ public void setFlowFilesSent(int flowFilesSent) {
+ this.flowFilesSent = flowFilesSent;
+ }
+
+ public long getBytesSent() {
+ return bytesSent;
+ }
+
+ public void setBytesSent(long bytesSent) {
+ this.bytesSent = bytesSent;
+ }
+
+ @Override
+ public ProcessorStatus clone() {
+ final ProcessorStatus clonedObj = new ProcessorStatus();
+ clonedObj.activeThreadCount = activeThreadCount;
+ clonedObj.bytesRead = bytesRead;
+ clonedObj.bytesWritten = bytesWritten;
+ clonedObj.flowFilesReceived = flowFilesReceived;
+ clonedObj.bytesReceived = bytesReceived;
+ clonedObj.flowFilesSent = flowFilesSent;
+ clonedObj.bytesSent = bytesSent;
+ clonedObj.groupId = groupId;
+ clonedObj.id = id;
+ clonedObj.inputBytes = inputBytes;
+ clonedObj.inputCount = inputCount;
+ clonedObj.invocations = invocations;
+ clonedObj.name = name;
+ clonedObj.outputBytes = outputBytes;
+ clonedObj.outputCount = outputCount;
+ clonedObj.processingNanos = processingNanos;
+ clonedObj.averageLineageDuration = averageLineageDuration;
+ clonedObj.flowFilesRemoved = flowFilesRemoved;
+ clonedObj.runStatus = runStatus;
+ clonedObj.type = type;
+ return clonedObj;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("ProcessorStatus [id=");
+ builder.append(id);
+ builder.append(", groupId=");
+ builder.append(groupId);
+ builder.append(", name=");
+ builder.append(name);
+ builder.append(", type=");
+ builder.append(type);
+ builder.append(", runStatus=");
+ builder.append(runStatus);
+ builder.append(", inputCount=");
+ builder.append(inputCount);
+ builder.append(", inputBytes=");
+ builder.append(inputBytes);
+ builder.append(", outputCount=");
+ builder.append(outputCount);
+ builder.append(", outputBytes=");
+ builder.append(outputBytes);
+ builder.append(", bytesRead=");
+ builder.append(bytesRead);
+ builder.append(", bytesWritten=");
+ builder.append(bytesWritten);
+ builder.append(", invocations=");
+ builder.append(invocations);
+ builder.append(", processingNanos=");
+ builder.append(processingNanos);
+ builder.append(", activeThreadCount=");
+ builder.append(activeThreadCount);
+ builder.append("]");
+ return builder.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/controller/status/RemoteProcessGroupStatus.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/status/RemoteProcessGroupStatus.java b/nifi-api/src/main/java/org/apache/nifi/controller/status/RemoteProcessGroupStatus.java
new file mode 100644
index 0000000..110972e
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/status/RemoteProcessGroupStatus.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller.status;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+
+/**
+ *
+ */
+public class RemoteProcessGroupStatus implements Cloneable {
+
+ private String id;
+ private String groupId;
+ private TransmissionStatus transmissionStatus;
+ private String uri;
+ private String name;
+ private Integer activeThreadCount;
+ private int sentCount;
+ private long sentContentSize;
+ private int receivedCount;
+ private long receivedContentSize;
+ private Integer activeRemotePortCount;
+ private Integer inactiveRemotePortCount;
+
+ private long averageLineageDuration;
+ private List<String> authorizationIssues = new ArrayList<>();
+
+ public String getTargetUri() {
+ return uri;
+ }
+
+ public void setTargetUri(String uri) {
+ this.uri = uri;
+ }
+
+ public TransmissionStatus getTransmissionStatus() {
+ return transmissionStatus;
+ }
+
+ public void setTransmissionStatus(TransmissionStatus transmissionStatus) {
+ this.transmissionStatus = transmissionStatus;
+ }
+
+ public String getGroupId() {
+ return groupId;
+ }
+
+ public void setGroupId(String groupId) {
+ this.groupId = groupId;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ public Integer getActiveThreadCount() {
+ return activeThreadCount;
+ }
+
+ public void setActiveThreadCount(Integer activeThreadCount) {
+ this.activeThreadCount = activeThreadCount;
+ }
+
+ public Integer getSentCount() {
+ return sentCount;
+ }
+
+ public void setSentCount(Integer sentCount) {
+ this.sentCount = sentCount;
+ }
+
+ public Long getSentContentSize() {
+ return sentContentSize;
+ }
+
+ public void setSentContentSize(Long sentContentSize) {
+ this.sentContentSize = sentContentSize;
+ }
+
+ public Integer getReceivedCount() {
+ return receivedCount;
+ }
+
+ public void setReceivedCount(Integer receivedCount) {
+ this.receivedCount = receivedCount;
+ }
+
+ public Long getReceivedContentSize() {
+ return receivedContentSize;
+ }
+
+ public void setReceivedContentSize(Long receivedContentSize) {
+ this.receivedContentSize = receivedContentSize;
+ }
+
+ public Integer getActiveRemotePortCount() {
+ return activeRemotePortCount;
+ }
+
+ public void setActiveRemotePortCount(Integer activeRemotePortCount) {
+ this.activeRemotePortCount = activeRemotePortCount;
+ }
+
+ public Integer getInactiveRemotePortCount() {
+ return inactiveRemotePortCount;
+ }
+
+ public void setInactiveRemotePortCount(Integer inactiveRemotePortCount) {
+ this.inactiveRemotePortCount = inactiveRemotePortCount;
+ }
+
+ public List<String> getAuthorizationIssues() {
+ return new ArrayList<>(authorizationIssues);
+ }
+
+ public void setAuthorizationIssues(List<String> authorizationIssues) {
+ this.authorizationIssues = new ArrayList<>(Objects.requireNonNull(authorizationIssues));
+ }
+
+ public long getAverageLineageDuration() {
+ return averageLineageDuration;
+ }
+
+ public void setAverageLineageDuration(final long millis) {
+ this.averageLineageDuration = millis;
+ }
+
+ public long getAverageLineageDuration(final TimeUnit timeUnit) {
+ return TimeUnit.MILLISECONDS.convert(averageLineageDuration, timeUnit);
+ }
+
+ public void setAverageLineageDuration(final long duration, final TimeUnit timeUnit) {
+ this.averageLineageDuration = timeUnit.toMillis(duration);
+ }
+
+ @Override
+ public RemoteProcessGroupStatus clone() {
+ final RemoteProcessGroupStatus clonedObj = new RemoteProcessGroupStatus();
+ clonedObj.id = id;
+ clonedObj.groupId = groupId;
+ clonedObj.name = name;
+ clonedObj.uri = uri;
+ clonedObj.activeThreadCount = activeThreadCount;
+ clonedObj.transmissionStatus = transmissionStatus;
+ clonedObj.sentCount = sentCount;
+ clonedObj.sentContentSize = sentContentSize;
+ clonedObj.receivedCount = receivedCount;
+ clonedObj.receivedContentSize = receivedContentSize;
+ clonedObj.activeRemotePortCount = activeRemotePortCount;
+ clonedObj.inactiveRemotePortCount = inactiveRemotePortCount;
+ clonedObj.averageLineageDuration = averageLineageDuration;
+ clonedObj.authorizationIssues = getAuthorizationIssues();
+ return clonedObj;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("RemoteProcessGroupStatus [id=");
+ builder.append(id);
+ builder.append(", groupId=");
+ builder.append(groupId);
+ builder.append(", name=");
+ builder.append(name);
+ builder.append(", uri=");
+ builder.append(uri);
+ builder.append(", activeThreadCount=");
+ builder.append(activeThreadCount);
+ builder.append(", transmissionStatus=");
+ builder.append(transmissionStatus);
+ builder.append(", sentCount=");
+ builder.append(sentCount);
+ builder.append(", sentContentSize=");
+ builder.append(sentContentSize);
+ builder.append(", receivedCount=");
+ builder.append(receivedCount);
+ builder.append(", receivedContentSize=");
+ builder.append(receivedContentSize);
+ builder.append(", activeRemotePortCount=");
+ builder.append(activeRemotePortCount);
+ builder.append(", inactiveRemotePortCount=");
+ builder.append(inactiveRemotePortCount);
+ builder.append(", authenticationIssues=");
+ builder.append(authorizationIssues);
+ builder.append("]");
+ return builder.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/controller/status/RunStatus.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/status/RunStatus.java b/nifi-api/src/main/java/org/apache/nifi/controller/status/RunStatus.java
new file mode 100644
index 0000000..1b7c43d
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/status/RunStatus.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller.status;
+
+/**
+ *
+ */
+public enum RunStatus {
+
+ Running,
+ Stopped,
+ Invalid,
+ Disabled;
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/controller/status/TransmissionStatus.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/status/TransmissionStatus.java b/nifi-api/src/main/java/org/apache/nifi/controller/status/TransmissionStatus.java
new file mode 100644
index 0000000..6d7eb12
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/status/TransmissionStatus.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller.status;
+
+public enum TransmissionStatus {
+
+ Transmitting,
+ NotTransmitting;
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/controller/status/history/ComponentStatusRepository.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/status/history/ComponentStatusRepository.java b/nifi-api/src/main/java/org/apache/nifi/controller/status/history/ComponentStatusRepository.java
new file mode 100644
index 0000000..4628a28
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/status/history/ComponentStatusRepository.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller.status.history;
+
+import java.util.Date;
+import java.util.List;
+
+import org.apache.nifi.controller.status.ConnectionStatus;
+import org.apache.nifi.controller.status.ProcessGroupStatus;
+import org.apache.nifi.controller.status.ProcessorStatus;
+import org.apache.nifi.controller.status.RemoteProcessGroupStatus;
+
+/**
+ * A repository for storing and retrieving components' historical status
+ * information
+ */
+public interface ComponentStatusRepository {
+
+ /**
+ * Captures the status information provided in the given report
+ *
+ * @param rootGroupStatus status of root group
+ */
+ void capture(ProcessGroupStatus rootGroupStatus);
+
+ /**
+ * Captures the status information provided in the given report, providing a
+ * timestamp that indicates the time at which the status report was
+ * generated. This can be used to replay historical values.
+ *
+ * @param rootGroupStatus status
+ * @param timestamp timestamp of capture
+ */
+ void capture(ProcessGroupStatus rootGroupStatus, Date timestamp);
+
+ /**
+ * @return the Date at which the latest capture was performed
+ */
+ Date getLastCaptureDate();
+
+ /**
+ * @param connectionId the ID of the Connection for which the Status is
+ * desired
+ * @param start the earliest date for which status information should be
+ * returned; if <code>null</code>, the start date should be assumed to be
+ * the beginning of time
+ * @param end the latest date for which status information should be
+ * returned; if <code>null</code>, the end date should be assumed to be the
+ * current time
+ * @param preferredDataPoints the preferred number of data points to return.
+ * If the date range is large, the total number of data points could be far
+ * too many to process. Therefore, this parameter allows the requestor to
+ * indicate how many samples to return.
+ * @return a {@link StatusHistory} that provides the status information
+ * about the Connection with the given ID during the given time period
+ */
+ StatusHistory getConnectionStatusHistory(String connectionId, Date start, Date end, int preferredDataPoints);
+
+ /**
+ * @param processGroupId of group to get status of
+ * @param start the earliest date for which status information should be
+ * returned; if <code>null</code>, the start date should be assumed to be
+ * the beginning of time
+ * @param end the latest date for which status information should be
+ * returned; if <code>null</code>, the end date should be assumed to be the
+ * current time
+ * @param preferredDataPoints the preferred number of data points to return.
+ * If the date range is large, the total number of data points could be far
+ * too many to process. Therefore, this parameter allows the requestor to
+ * indicate how many samples to return.
+ * @return a {@link StatusHistory} that provides the status information
+ * about the Process Group with the given ID during the given time period
+ */
+ StatusHistory getProcessGroupStatusHistory(String processGroupId, Date start, Date end, int preferredDataPoints);
+
+ /**
+ * @param processorId to get status of
+ * @param start the earliest date for which status information should be
+ * returned; if <code>null</code>, the start date should be assumed to be
+ * the beginning of time
+ * @param end the latest date for which status information should be
+ * returned; if <code>null</code>, the end date should be assumed to be the
+ * current time
+ * @param preferredDataPoints the preferred number of data points to return.
+ * If the date range is large, the total number of data points could be far
+ * too many to process. Therefore, this parameter allows the requestor to
+ * indicate how many samples to return.
+ * @return a {@link StatusHistory} that provides the status information
+ * about the Processor with the given ID during the given time period
+ */
+ StatusHistory getProcessorStatusHistory(String processorId, Date start, Date end, int preferredDataPoints);
+
+ /**
+ * @param remoteGroupId to get history of
+ * @param start the earliest date for which status information should be
+ * returned; if <code>null</code>, the start date should be assumed to be
+ * the beginning of time
+ * @param end the latest date for which status information should be
+ * returned; if <code>null</code>, the end date should be assumed to be the
+ * current time
+ * @param preferredDataPoints the preferred number of data points to return.
+ * If the date range is large, the total number of data points could be far
+ * too many to process. Therefore, this parameter allows the requestor to
+ * indicate how many samples to return.
+ * @return a {@link StatusHistory} that provides the status information
+ * about the Remote Process Group with the given ID during the given time
+ * period
+ */
+ StatusHistory getRemoteProcessGroupStatusHistory(String remoteGroupId, Date start, Date end, int preferredDataPoints);
+
+ /**
+ * @return a List of all {@link MetricDescriptor}s that are applicable to
+ * Process Groups
+ */
+ List<MetricDescriptor<ProcessGroupStatus>> getProcessGroupMetricDescriptors();
+
+ /**
+ * @return a List of all {@link MetricDescriptor}s that are applicable to
+ * Processors
+ */
+ List<MetricDescriptor<ProcessorStatus>> getProcessorMetricDescriptors();
+
+ /**
+ * @return a List of all {@link MetricDescriptor}s that are applicable to
+ * Remote Process Groups
+ */
+ List<MetricDescriptor<RemoteProcessGroupStatus>> getRemoteProcessGroupMetricDescriptors();
+
+ /**
+ * @return a List of all {@link MetricDescriptor}s that are applicable to
+ * Connections
+ */
+ List<MetricDescriptor<ConnectionStatus>> getConnectionMetricDescriptors();
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/controller/status/history/MetricDescriptor.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/status/history/MetricDescriptor.java b/nifi-api/src/main/java/org/apache/nifi/controller/status/history/MetricDescriptor.java
new file mode 100644
index 0000000..8fdce05
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/status/history/MetricDescriptor.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller.status.history;
+
+/**
+ * Describes a particular metric that is derived from a Status History
+ *
+ * @param <T> type of metric
+ */
+public interface MetricDescriptor<T> {
+
+ public enum Formatter {
+
+ COUNT,
+ DURATION,
+ DATA_SIZE
+ };
+
+ /**
+ * Specifies how the values should be formatted
+ *
+ * @return formatter for values
+ */
+ Formatter getFormatter();
+
+ /**
+ * @return a human-readable description of the field
+ */
+ String getDescription();
+
+ /**
+ * @return a human-readable label for the field
+ */
+ String getLabel();
+
+ /**
+ * @return the name of a field
+ */
+ String getField();
+
+ /**
+ * @return a {@link ValueMapper} that can be used to extract a value for the
+ * status history
+ */
+ ValueMapper<T> getValueFunction();
+
+ /**
+ * @return a {@link ValueReducer} that can reduce multiple StatusSnapshots
+ * into a single Long value
+ */
+ ValueReducer<StatusSnapshot, Long> getValueReducer();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/controller/status/history/StatusHistory.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/status/history/StatusHistory.java b/nifi-api/src/main/java/org/apache/nifi/controller/status/history/StatusHistory.java
new file mode 100644
index 0000000..f1bb946
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/status/history/StatusHistory.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller.status.history;
+
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Represents a collection of historical status values for a component
+ */
+public interface StatusHistory {
+
+ /**
+ * @return a Date indicating when this report was generated
+ */
+ Date getDateGenerated();
+
+ /**
+ * @return a Map of component field names and their values. The order in
+ * which these values are displayed is dependent on the natural ordering of
+ * the Map returned
+ */
+ Map<String, String> getComponentDetails();
+
+ /**
+ * @return List of snapshots for a given component
+ */
+ List<StatusSnapshot> getStatusSnapshots();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/controller/status/history/StatusSnapshot.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/status/history/StatusSnapshot.java b/nifi-api/src/main/java/org/apache/nifi/controller/status/history/StatusSnapshot.java
new file mode 100644
index 0000000..551ceb2
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/status/history/StatusSnapshot.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller.status.history;
+
+import java.util.Date;
+import java.util.Map;
+
+/**
+ * A StatusSnapshot represents a Component's status report at some point in time
+ */
+public interface StatusSnapshot {
+
+ /**
+ * @return the point in time for which the status values were obtained
+ */
+ Date getTimestamp();
+
+ /**
+ * @return a Map of MetricDescriptor to value
+ */
+ Map<MetricDescriptor<?>, Long> getStatusMetrics();
+
+ /**
+ * @return a {@link ValueReducer} that is capable of merging multiple
+ * StatusSnapshot objects into a single one
+ */
+ ValueReducer<StatusSnapshot, StatusSnapshot> getValueReducer();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/controller/status/history/ValueMapper.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/status/history/ValueMapper.java b/nifi-api/src/main/java/org/apache/nifi/controller/status/history/ValueMapper.java
new file mode 100644
index 0000000..8000b3a
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/status/history/ValueMapper.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller.status.history;
+
+public interface ValueMapper<S> {
+
+ Long getValue(S status);
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/controller/status/history/ValueReducer.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/status/history/ValueReducer.java b/nifi-api/src/main/java/org/apache/nifi/controller/status/history/ValueReducer.java
new file mode 100644
index 0000000..0427da7
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/status/history/ValueReducer.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller.status.history;
+
+import java.util.List;
+
+public interface ValueReducer<T, R> {
+
+ R reduce(List<T> values);
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/events/EventReporter.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/events/EventReporter.java b/nifi-api/src/main/java/org/apache/nifi/events/EventReporter.java
new file mode 100644
index 0000000..d645d60
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/events/EventReporter.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.events;
+
+import java.io.Serializable;
+
+import org.apache.nifi.reporting.Severity;
+
+/**
+ * Implementations MUST be thread-safe
+ */
+public interface EventReporter extends Serializable {
+
+ void reportEvent(Severity severity, String category, String message);
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/expression/AttributeExpression.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/expression/AttributeExpression.java b/nifi-api/src/main/java/org/apache/nifi/expression/AttributeExpression.java
new file mode 100644
index 0000000..ed409ea
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/expression/AttributeExpression.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.expression;
+
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.exception.ProcessException;
+
+public interface AttributeExpression {
+
+ /**
+ * @return Evaluates the expression without providing any FlowFile Attributes. This
+ * will evaluate the expression based only on System Properties and JVM
+ * Environment properties
+ * @throws ProcessException if unable to evaluate
+ */
+ String evaluate() throws ProcessException;
+
+ /**
+ * Evaluates the expression without providing any FlowFile Attributes. This
+ * will evaluate the expression based only on System Properties and JVM
+ * Environment properties but allows the values to be decorated
+ *
+ * @param decorator for attribute value
+ * @return evaluated value
+ * @throws ProcessException if failure in evaluation
+ */
+ String evaluate(AttributeValueDecorator decorator) throws ProcessException;
+
+ /**
+ * Evaluates the expression, providing access to the attributes, file size,
+ * id, etc. of the given FlowFile, as well as System Properties and JVM
+ * Environment properties
+ *
+ * @param flowFile to evaluate
+ * @return evaluated value
+ * @throws ProcessException if failure evaluating
+ */
+ String evaluate(FlowFile flowFile) throws ProcessException;
+
+ /**
+ * Evaluates the expression, providing access to the attributes, file size,
+ * id, etc. of the given FlowFile, as well as System Properties and JVM
+ * Environment properties and allows the values to be decorated
+ *
+ * @param flowFile to evaluate
+ * @param decorator for evaluation
+ * @return evaluated value
+ * @throws ProcessException if failed to evaluate
+ */
+ String evaluate(FlowFile flowFile, AttributeValueDecorator decorator) throws ProcessException;
+
+ /**
+ * @return the type that is returned by the Expression
+ */
+ ResultType getResultType();
+
+ public static enum ResultType {
+
+ STRING, BOOLEAN, NUMBER, DATE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/expression/AttributeValueDecorator.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/expression/AttributeValueDecorator.java b/nifi-api/src/main/java/org/apache/nifi/expression/AttributeValueDecorator.java
new file mode 100644
index 0000000..4cea248
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/expression/AttributeValueDecorator.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.expression;
+
+public interface AttributeValueDecorator {
+
+ /**
+ * Decorates the value of a FlowFile Attribute or System/JVM property in
+ * some way
+ *
+ * @param attributeValue to decorate
+ * @return decorated value
+ */
+ String decorate(String attributeValue);
+}
[37/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/Evaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/Evaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/Evaluator.java
new file mode 100644
index 0000000..86f95a8
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/Evaluator.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation;
+
+import java.util.Map;
+
+import org.apache.nifi.expression.AttributeExpression.ResultType;
+
+public interface Evaluator<T> {
+
+ QueryResult<T> evaluate(Map<String, String> attributes);
+
+ ResultType getResultType();
+
+ int getEvaluationsRemaining();
+
+ Evaluator<?> getSubjectEvaluator();
+
+ /**
+ * Sets the token that was used in the query to cause this Evaluator to be created
+ *
+ * @param token the token that caused this Evaluator to be created
+ */
+ void setToken(String token);
+
+ /**
+ * @return the token that caused this Evaluator to be created
+ */
+ String getToken();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/NumberEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/NumberEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/NumberEvaluator.java
new file mode 100644
index 0000000..ec3adac
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/NumberEvaluator.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation;
+
+import org.apache.nifi.expression.AttributeExpression.ResultType;
+
+public abstract class NumberEvaluator implements Evaluator<Long> {
+ private String token;
+
+ @Override
+ public ResultType getResultType() {
+ return ResultType.NUMBER;
+ }
+
+ @Override
+ public int getEvaluationsRemaining() {
+ return 0;
+ }
+
+ @Override
+ public String getToken() {
+ return token;
+ }
+
+ @Override
+ public void setToken(final String token) {
+ this.token = token;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/NumberQueryResult.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/NumberQueryResult.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/NumberQueryResult.java
new file mode 100644
index 0000000..fc3c961
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/NumberQueryResult.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation;
+
+import org.apache.nifi.expression.AttributeExpression.ResultType;
+
+public class NumberQueryResult implements QueryResult<Long> {
+
+ private final Long value;
+
+ public NumberQueryResult(final Long value) {
+ this.value = value;
+ }
+
+ @Override
+ public Long getValue() {
+ return value;
+ }
+
+ @Override
+ public ResultType getResultType() {
+ return ResultType.NUMBER;
+ }
+
+ @Override
+ public String toString() {
+ return String.valueOf(getValue());
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/QueryResult.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/QueryResult.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/QueryResult.java
new file mode 100644
index 0000000..56bd76a
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/QueryResult.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation;
+
+import org.apache.nifi.expression.AttributeExpression.ResultType;
+
+public interface QueryResult<T> {
+
+ T getValue();
+
+ ResultType getResultType();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/StringEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/StringEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/StringEvaluator.java
new file mode 100644
index 0000000..c6f2b79
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/StringEvaluator.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation;
+
+import org.apache.nifi.expression.AttributeExpression.ResultType;
+
+public abstract class StringEvaluator implements Evaluator<String> {
+ private String token;
+
+ @Override
+ public ResultType getResultType() {
+ return ResultType.STRING;
+ }
+
+ @Override
+ public int getEvaluationsRemaining() {
+ return 0;
+ }
+
+ @Override
+ public String getToken() {
+ return token;
+ }
+
+ @Override
+ public void setToken(final String token) {
+ this.token = token;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/StringQueryResult.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/StringQueryResult.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/StringQueryResult.java
new file mode 100644
index 0000000..c010bb1
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/StringQueryResult.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation;
+
+import org.apache.nifi.expression.AttributeExpression.ResultType;
+
+public class StringQueryResult implements QueryResult<String> {
+
+ private final String value;
+
+ public StringQueryResult(final String value) {
+ this.value = value;
+ }
+
+ @Override
+ public String getValue() {
+ return value;
+ }
+
+ @Override
+ public ResultType getResultType() {
+ return ResultType.STRING;
+ }
+
+ @Override
+ public String toString() {
+ return String.valueOf(getValue());
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/BooleanCastEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/BooleanCastEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/BooleanCastEvaluator.java
new file mode 100644
index 0000000..a88b72b
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/BooleanCastEvaluator.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.cast;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator;
+
+public class BooleanCastEvaluator extends BooleanEvaluator {
+
+ private final StringEvaluator subjectEvaluator;
+
+ public BooleanCastEvaluator(final StringEvaluator subjectEvaluator) {
+ this.subjectEvaluator = subjectEvaluator;
+ }
+
+ @Override
+ public QueryResult<Boolean> evaluate(final Map<String, String> attributes) {
+ final QueryResult<String> result = subjectEvaluator.evaluate(attributes);
+ if (result.getValue() == null) {
+ return new BooleanQueryResult(null);
+ }
+
+ return new BooleanQueryResult(Boolean.valueOf(result.getValue().trim()));
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subjectEvaluator;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/DateCastEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/DateCastEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/DateCastEvaluator.java
new file mode 100644
index 0000000..96e74ef
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/DateCastEvaluator.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.cast;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Locale;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.nifi.attribute.expression.language.evaluation.DateEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.DateQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult;
+import org.apache.nifi.attribute.expression.language.exception.AttributeExpressionLanguageException;
+import org.apache.nifi.attribute.expression.language.exception.AttributeExpressionLanguageParsingException;
+import org.apache.nifi.expression.AttributeExpression.ResultType;
+
+public class DateCastEvaluator extends DateEvaluator {
+
+ public static final String DATE_TO_STRING_FORMAT = "EEE MMM dd HH:mm:ss zzz yyyy";
+ public static final Pattern DATE_TO_STRING_PATTERN = Pattern.compile("(?:[a-zA-Z]{3} ){2}\\d{2} \\d{2}\\:\\d{2}\\:\\d{2} (?:.*?) \\d{4}");
+
+ public static final String ALTERNATE_FORMAT_WITHOUT_MILLIS = "yyyy/MM/dd HH:mm:ss";
+ public static final String ALTERNATE_FORMAT_WITH_MILLIS = "yyyy/MM/dd HH:mm:ss.SSS";
+ public static final Pattern ALTERNATE_PATTERN = Pattern.compile("\\d{4}/\\d{2}/\\d{2} \\d{2}\\:\\d{2}\\:\\d{2}(\\.\\d{3})?");
+
+ public static final Pattern NUMBER_PATTERN = Pattern.compile("\\d+");
+
+ private final Evaluator<?> subjectEvaluator;
+
+ public DateCastEvaluator(final Evaluator<?> subjectEvaluator) {
+ if (subjectEvaluator.getResultType() == ResultType.BOOLEAN) {
+ throw new AttributeExpressionLanguageParsingException("Cannot implicitly convert Data Type " + subjectEvaluator.getResultType() + " to " + ResultType.DATE);
+ }
+
+ this.subjectEvaluator = subjectEvaluator;
+ }
+
+ @Override
+ public QueryResult<Date> evaluate(final Map<String, String> attributes) {
+ final QueryResult<?> result = subjectEvaluator.evaluate(attributes);
+ if (result.getValue() == null) {
+ return new DateQueryResult(null);
+ }
+
+ switch (result.getResultType()) {
+ case DATE:
+ return (DateQueryResult) result;
+ case STRING:
+ final String value = ((StringQueryResult) result).getValue().trim();
+ if (DATE_TO_STRING_PATTERN.matcher(value).matches()) {
+ final SimpleDateFormat sdf = new SimpleDateFormat(DATE_TO_STRING_FORMAT, Locale.US);
+
+ try {
+ final Date date = sdf.parse(value);
+ return new DateQueryResult(date);
+ } catch (final ParseException pe) {
+ final String details = "Format: '" + DATE_TO_STRING_FORMAT + "' Value: '" + value + "'";
+ throw new AttributeExpressionLanguageException("Could not parse date using " + details, pe);
+ }
+ } else if (NUMBER_PATTERN.matcher(value).matches()) {
+ return new DateQueryResult(new Date(Long.valueOf(value)));
+ } else {
+ final Matcher altMatcher = ALTERNATE_PATTERN.matcher(value);
+ if (altMatcher.matches()) {
+ final String millisValue = altMatcher.group(1);
+
+ final String format;
+ if (millisValue == null) {
+ format = ALTERNATE_FORMAT_WITHOUT_MILLIS;
+ } else {
+ format = ALTERNATE_FORMAT_WITH_MILLIS;
+ }
+
+ final SimpleDateFormat sdf = new SimpleDateFormat(format, Locale.US);
+
+ try {
+ final Date date = sdf.parse(value);
+ return new DateQueryResult(date);
+ } catch (final ParseException pe) {
+ throw new AttributeExpressionLanguageException("Could not parse input as date", pe);
+ }
+ } else {
+ throw new AttributeExpressionLanguageException("Could not implicitly convert input to DATE: " + value);
+ }
+ }
+ case NUMBER:
+ return new DateQueryResult(new Date((Long) result.getValue()));
+ default:
+ return new DateQueryResult(null);
+ }
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subjectEvaluator;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/NumberCastEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/NumberCastEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/NumberCastEvaluator.java
new file mode 100644
index 0000000..0447d48
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/NumberCastEvaluator.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.cast;
+
+import java.util.Map;
+import java.util.regex.Pattern;
+
+import org.apache.nifi.attribute.expression.language.evaluation.DateQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.NumberEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.NumberQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult;
+import org.apache.nifi.attribute.expression.language.exception.AttributeExpressionLanguageParsingException;
+import org.apache.nifi.expression.AttributeExpression.ResultType;
+
+public class NumberCastEvaluator extends NumberEvaluator {
+
+ private final Evaluator<?> subjectEvaluator;
+ private static final Pattern NUMBER_PATTERN = Pattern.compile("-?\\d+");
+
+ public NumberCastEvaluator(final Evaluator<?> subjectEvaluator) {
+ if (subjectEvaluator.getResultType() == ResultType.BOOLEAN) {
+ throw new AttributeExpressionLanguageParsingException("Cannot implicitly convert Data Type " + subjectEvaluator.getResultType() + " to " + ResultType.NUMBER);
+ }
+ this.subjectEvaluator = subjectEvaluator;
+ }
+
+ @Override
+ public QueryResult<Long> evaluate(final Map<String, String> attributes) {
+ final QueryResult<?> result = subjectEvaluator.evaluate(attributes);
+ if (result.getValue() == null) {
+ return new NumberQueryResult(null);
+ }
+
+ switch (result.getResultType()) {
+ case NUMBER:
+ return (NumberQueryResult) result;
+ case STRING:
+ final String trimmed = ((StringQueryResult) result).getValue().trim();
+ if (NUMBER_PATTERN.matcher(trimmed).matches()) {
+ return new NumberQueryResult(Long.valueOf(trimmed));
+ } else {
+ return new NumberQueryResult(null);
+ }
+ case DATE:
+ return new NumberQueryResult(((DateQueryResult) result).getValue().getTime());
+ default:
+ return new NumberQueryResult(null);
+ }
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subjectEvaluator;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/StringCastEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/StringCastEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/StringCastEvaluator.java
new file mode 100644
index 0000000..1a3732a
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/StringCastEvaluator.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.cast;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult;
+
+public class StringCastEvaluator extends StringEvaluator {
+
+ private final Evaluator<?> subjectEvaluator;
+
+ public StringCastEvaluator(final Evaluator<?> subjectEvaluator) {
+ this.subjectEvaluator = subjectEvaluator;
+ }
+
+ @Override
+ public QueryResult<String> evaluate(final Map<String, String> attributes) {
+ final QueryResult<?> result = subjectEvaluator.evaluate(attributes);
+ if (result.getValue() == null) {
+ return new StringQueryResult(null);
+ }
+
+ return new StringQueryResult(String.valueOf(result.getValue()));
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subjectEvaluator;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/AndEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/AndEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/AndEvaluator.java
new file mode 100644
index 0000000..232fc26
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/AndEvaluator.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+
+public class AndEvaluator extends BooleanEvaluator {
+
+ private final Evaluator<Boolean> subjectEvaluator;
+ private final Evaluator<Boolean> rhsEvaluator;
+
+ public AndEvaluator(final Evaluator<Boolean> subjectEvaluator, final Evaluator<Boolean> rhsEvaluator) {
+ this.subjectEvaluator = subjectEvaluator;
+ this.rhsEvaluator = rhsEvaluator;
+ }
+
+ @Override
+ public QueryResult<Boolean> evaluate(final Map<String, String> attributes) {
+ final QueryResult<Boolean> subjectValue = subjectEvaluator.evaluate(attributes);
+ if (subjectValue == null) {
+ return new BooleanQueryResult(null);
+ }
+
+ if (Boolean.FALSE.equals(subjectValue.getValue())) {
+ return new BooleanQueryResult(false);
+ }
+
+ final QueryResult<Boolean> rhsValue = rhsEvaluator.evaluate(attributes);
+ if (rhsValue == null) {
+ return new BooleanQueryResult(false);
+ }
+
+ return new BooleanQueryResult(rhsValue.getValue());
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subjectEvaluator;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/AppendEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/AppendEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/AppendEvaluator.java
new file mode 100644
index 0000000..80f1974
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/AppendEvaluator.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult;
+
+public class AppendEvaluator extends StringEvaluator {
+
+ private final Evaluator<String> subject;
+ private final Evaluator<String> appendEvaluator;
+
+ public AppendEvaluator(final Evaluator<String> subject, final Evaluator<String> append) {
+ this.subject = subject;
+ this.appendEvaluator = append;
+ }
+
+ @Override
+ public QueryResult<String> evaluate(final Map<String, String> attributes) {
+ final String subjectValue = subject.evaluate(attributes).getValue();
+ final String appendValue = appendEvaluator.evaluate(attributes).getValue();
+
+ final String result = (subjectValue == null ? "" : subjectValue)
+ + (appendValue == null ? "" : appendValue);
+ return new StringQueryResult(result);
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/AttributeEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/AttributeEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/AttributeEvaluator.java
new file mode 100644
index 0000000..a0695a9
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/AttributeEvaluator.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult;
+
+public class AttributeEvaluator extends StringEvaluator {
+
+ private final Evaluator<String> nameEvaluator;
+
+ public AttributeEvaluator(final Evaluator<String> nameEvaluator) {
+ this.nameEvaluator = nameEvaluator;
+ }
+
+ @Override
+ public QueryResult<String> evaluate(final Map<String, String> attributes) {
+ final String nameValue = nameEvaluator.evaluate(attributes).getValue();
+ final String attributeValue = attributes.get(nameValue);
+ return new StringQueryResult(attributeValue);
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ContainsEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ContainsEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ContainsEvaluator.java
new file mode 100644
index 0000000..54170ba
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ContainsEvaluator.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+
+public class ContainsEvaluator extends BooleanEvaluator {
+
+ private final Evaluator<String> subject;
+ private final Evaluator<String> search;
+
+ public ContainsEvaluator(final Evaluator<String> subject, final Evaluator<String> search) {
+ this.subject = subject;
+ this.search = search;
+ }
+
+ @Override
+ public QueryResult<Boolean> evaluate(final Map<String, String> attributes) {
+ final String subjectValue = subject.evaluate(attributes).getValue();
+ if (subjectValue == null) {
+ return new BooleanQueryResult(false);
+ }
+
+ final String searchString = search.evaluate(attributes).getValue();
+ return new BooleanQueryResult(searchString == null ? false : subjectValue.contains(searchString));
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/DateToNumberEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/DateToNumberEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/DateToNumberEvaluator.java
new file mode 100644
index 0000000..0594c4a
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/DateToNumberEvaluator.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Date;
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.DateEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.NumberEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.NumberQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+
+public class DateToNumberEvaluator extends NumberEvaluator {
+
+ private final DateEvaluator subjectEvaluator;
+
+ public DateToNumberEvaluator(final DateEvaluator subjectEvaluator) {
+ this.subjectEvaluator = subjectEvaluator;
+ }
+
+ @Override
+ public QueryResult<Long> evaluate(final Map<String, String> attributes) {
+ final QueryResult<Date> subjectResult = subjectEvaluator.evaluate(attributes);
+ if (subjectResult.getValue() == null) {
+ return new NumberQueryResult(null);
+ }
+
+ return new NumberQueryResult(subjectResult.getValue().getTime());
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subjectEvaluator;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/DivideEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/DivideEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/DivideEvaluator.java
new file mode 100644
index 0000000..12ab37f
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/DivideEvaluator.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.NumberEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.NumberQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+
+public class DivideEvaluator extends NumberEvaluator {
+
+ private final Evaluator<Long> subject;
+ private final Evaluator<Long> divideValue;
+
+ public DivideEvaluator(final Evaluator<Long> subject, final Evaluator<Long> divideValue) {
+ this.subject = subject;
+ this.divideValue = divideValue;
+ }
+
+ @Override
+ public QueryResult<Long> evaluate(final Map<String, String> attributes) {
+ final Long subjectValue = subject.evaluate(attributes).getValue();
+ if (subjectValue == null) {
+ return new NumberQueryResult(null);
+ }
+
+ final Long divide = divideValue.evaluate(attributes).getValue();
+ if (divide == null) {
+ return new NumberQueryResult(null);
+ }
+
+ final long result = subjectValue / divide;
+ return new NumberQueryResult(result);
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/EndsWithEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/EndsWithEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/EndsWithEvaluator.java
new file mode 100644
index 0000000..13bc396
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/EndsWithEvaluator.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+
+public class EndsWithEvaluator extends BooleanEvaluator {
+
+ private final Evaluator<String> subject;
+ private final Evaluator<String> search;
+
+ public EndsWithEvaluator(final Evaluator<String> subject, final Evaluator<String> search) {
+ this.subject = subject;
+ this.search = search;
+ }
+
+ @Override
+ public QueryResult<Boolean> evaluate(final Map<String, String> attributes) {
+ final String subjectValue = subject.evaluate(attributes).getValue();
+ if (subjectValue == null) {
+ return new BooleanQueryResult(false);
+ }
+
+ final String searchString = search.evaluate(attributes).getValue();
+ return new BooleanQueryResult(searchString == null ? false : subjectValue.endsWith(searchString));
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/EqualsEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/EqualsEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/EqualsEvaluator.java
new file mode 100644
index 0000000..ec67b0f
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/EqualsEvaluator.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Date;
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.expression.AttributeExpression.ResultType;
+
+public class EqualsEvaluator extends BooleanEvaluator {
+
+ private final Evaluator<?> subject;
+ private final Evaluator<?> compareTo;
+
+ public EqualsEvaluator(final Evaluator<?> subject, final Evaluator<?> compareTo) {
+ this.subject = subject;
+ this.compareTo = compareTo;
+ }
+
+ @Override
+ public QueryResult<Boolean> evaluate(final Map<String, String> attributes) {
+ final Object a = subject.evaluate(attributes).getValue();
+ if (a == null) {
+ return new BooleanQueryResult(false);
+ }
+
+ final Object b = compareTo.evaluate(attributes).getValue();
+ if (b == null) {
+ return new BooleanQueryResult(false);
+ }
+
+ if (subject.getResultType() == compareTo.getResultType()) {
+ return new BooleanQueryResult(a.equals(b));
+ }
+
+ final String normalizedSubjectValue = normalizeValue(subject.getResultType(), a);
+ if (normalizedSubjectValue == null) {
+ return new BooleanQueryResult(false);
+ }
+
+ final String normalizedCompareToValue = normalizeValue(compareTo.getResultType(), b);
+ if (normalizedCompareToValue == null) {
+ return new BooleanQueryResult(false);
+ }
+
+ return new BooleanQueryResult(normalizedSubjectValue.equals(normalizedCompareToValue));
+ }
+
+ private String normalizeValue(final ResultType type, final Object value) {
+ if (value == null) {
+ return null;
+ }
+
+ switch (type) {
+ case STRING:
+ return (String) value;
+ case DATE:
+ return String.valueOf(((Date) value).getTime());
+ case BOOLEAN:
+ case NUMBER:
+ default:
+ return String.valueOf(value);
+ }
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/EqualsIgnoreCaseEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/EqualsIgnoreCaseEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/EqualsIgnoreCaseEvaluator.java
new file mode 100644
index 0000000..344395b
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/EqualsIgnoreCaseEvaluator.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+
+public class EqualsIgnoreCaseEvaluator extends BooleanEvaluator {
+
+ private final Evaluator<?> subject;
+ private final Evaluator<?> compareTo;
+
+ public EqualsIgnoreCaseEvaluator(final Evaluator<?> subject, final Evaluator<?> compareTo) {
+ this.subject = subject;
+ this.compareTo = compareTo;
+ }
+
+ @Override
+ public QueryResult<Boolean> evaluate(final Map<String, String> attributes) {
+ final Object a = subject.evaluate(attributes).getValue();
+ if (a == null) {
+ return new BooleanQueryResult(false);
+ }
+
+ final Object b = compareTo.evaluate(attributes).getValue();
+ if (b == null) {
+ return new BooleanQueryResult(false);
+ }
+
+ if (a instanceof String && b instanceof String) {
+ return new BooleanQueryResult(((String) a).equalsIgnoreCase((String) b));
+ }
+
+ return new BooleanQueryResult(a.equals(b));
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/FindEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/FindEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/FindEvaluator.java
new file mode 100644
index 0000000..2a3f1ae
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/FindEvaluator.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+import java.util.regex.Pattern;
+
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.literals.StringLiteralEvaluator;
+
+public class FindEvaluator extends BooleanEvaluator {
+
+ private final Evaluator<String> subject;
+ private final Evaluator<String> search;
+
+ private final Pattern compiledPattern;
+
+ public FindEvaluator(final Evaluator<String> subject, final Evaluator<String> search) {
+ this.subject = subject;
+ this.search = search;
+
+ // if the search string is a literal, we don't need to evaluate it each time; we can just
+ // pre-compile it. Otherwise, it must be compiled every time.
+ if (search instanceof StringLiteralEvaluator) {
+ this.compiledPattern = Pattern.compile(search.evaluate(null).getValue());
+ } else {
+ this.compiledPattern = null;
+ }
+ }
+
+ @Override
+ public QueryResult<Boolean> evaluate(final Map<String, String> attributes) {
+ final String subjectValue = subject.evaluate(attributes).getValue();
+ if (subjectValue == null) {
+ return new BooleanQueryResult(false);
+ }
+ final Pattern pattern;
+ if (compiledPattern == null) {
+ pattern = Pattern.compile(search.evaluate(attributes).getValue());
+ } else {
+ pattern = compiledPattern;
+ }
+
+ final boolean found = pattern.matcher(subjectValue).find();
+
+ return new BooleanQueryResult(found);
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/FormatEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/FormatEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/FormatEvaluator.java
new file mode 100644
index 0000000..717cbd5
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/FormatEvaluator.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Locale;
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.DateEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult;
+
+public class FormatEvaluator extends StringEvaluator {
+
+ private final DateEvaluator subject;
+ private final Evaluator<String> format;
+
+ public FormatEvaluator(final DateEvaluator subject, final Evaluator<String> format) {
+ this.subject = subject;
+ this.format = format;
+ }
+
+ @Override
+ public QueryResult<String> evaluate(final Map<String, String> attributes) {
+ final Date subjectValue = subject.evaluate(attributes).getValue();
+ if (subjectValue == null) {
+ return new StringQueryResult(null);
+ }
+
+ final QueryResult<String> formatResult = format.evaluate(attributes);
+ final String format = formatResult.getValue();
+ if (format == null) {
+ return null;
+ }
+
+ return new StringQueryResult(new SimpleDateFormat(format, Locale.US).format(subjectValue));
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/GreaterThanEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/GreaterThanEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/GreaterThanEvaluator.java
new file mode 100644
index 0000000..8850225
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/GreaterThanEvaluator.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+
+public class GreaterThanEvaluator extends BooleanEvaluator {
+
+ private final Evaluator<Long> subject;
+ private final Evaluator<Long> comparison;
+
+ public GreaterThanEvaluator(final Evaluator<Long> subject, final Evaluator<Long> comparison) {
+ this.subject = subject;
+ this.comparison = comparison;
+ }
+
+ @Override
+ public QueryResult<Boolean> evaluate(final Map<String, String> attributes) {
+ final Long subjectValue = subject.evaluate(attributes).getValue();
+ if (subjectValue == null) {
+ return new BooleanQueryResult(false);
+ }
+
+ final Long comparisonValue = comparison.evaluate(attributes).getValue();
+ if (comparisonValue == null) {
+ return new BooleanQueryResult(false);
+ }
+
+ return new BooleanQueryResult(subjectValue > comparisonValue);
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/GreaterThanOrEqualEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/GreaterThanOrEqualEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/GreaterThanOrEqualEvaluator.java
new file mode 100644
index 0000000..ff71335
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/GreaterThanOrEqualEvaluator.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+
+public class GreaterThanOrEqualEvaluator extends BooleanEvaluator {
+
+ private final Evaluator<Long> subject;
+ private final Evaluator<Long> comparison;
+
+ public GreaterThanOrEqualEvaluator(final Evaluator<Long> subject, final Evaluator<Long> comparison) {
+ this.subject = subject;
+ this.comparison = comparison;
+ }
+
+ @Override
+ public QueryResult<Boolean> evaluate(final Map<String, String> attributes) {
+ final Long subjectValue = subject.evaluate(attributes).getValue();
+ if (subjectValue == null) {
+ return new BooleanQueryResult(false);
+ }
+
+ final Long comparisonValue = comparison.evaluate(attributes).getValue();
+ if (comparisonValue == null) {
+ return new BooleanQueryResult(false);
+ }
+
+ return new BooleanQueryResult(subjectValue >= comparisonValue);
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/HostnameEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/HostnameEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/HostnameEvaluator.java
new file mode 100644
index 0000000..c0b1134
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/HostnameEvaluator.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult;
+
+public class HostnameEvaluator extends StringEvaluator {
+
+ private final StringQueryResult hostname;
+
+ public HostnameEvaluator(final boolean preferFQDN) throws UnknownHostException {
+ String address;
+
+ if (preferFQDN) {
+ try {
+ address = InetAddress.getLocalHost().getCanonicalHostName();
+ } catch (final Exception e) {
+ address = InetAddress.getLocalHost().getHostName();
+ }
+ } else {
+ address = InetAddress.getLocalHost().getHostName();
+ }
+
+ hostname = new StringQueryResult(address);
+ }
+
+ @Override
+ public QueryResult<String> evaluate(final Map<String, String> attributes) {
+ return hostname;
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/IPEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/IPEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/IPEvaluator.java
new file mode 100644
index 0000000..21f9fe8
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/IPEvaluator.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult;
+
+public class IPEvaluator extends StringEvaluator {
+
+ private final StringQueryResult ipAddress;
+
+ public IPEvaluator() throws UnknownHostException {
+ ipAddress = new StringQueryResult(InetAddress.getLocalHost().getHostAddress());
+ }
+
+ @Override
+ public QueryResult<String> evaluate(final Map<String, String> attributes) {
+ return ipAddress;
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/IndexOfEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/IndexOfEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/IndexOfEvaluator.java
new file mode 100644
index 0000000..c4bc03e
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/IndexOfEvaluator.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.NumberEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.NumberQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+
+public class IndexOfEvaluator extends NumberEvaluator {
+
+ private final Evaluator<String> subject;
+ private final Evaluator<String> indexEvaluator;
+
+ public IndexOfEvaluator(final Evaluator<String> subject, final Evaluator<String> indexEvaluator) {
+ this.subject = subject;
+ this.indexEvaluator = indexEvaluator;
+ }
+
+ @Override
+ public QueryResult<Long> evaluate(final Map<String, String> attributes) {
+ final String subjectValue = subject.evaluate(attributes).getValue();
+ if (subjectValue == null) {
+ return new NumberQueryResult(-1L);
+ }
+ final String indexEvalValue = indexEvaluator.evaluate(attributes).getValue();
+
+ return new NumberQueryResult((long) subjectValue.indexOf(indexEvalValue));
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/IsEmptyEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/IsEmptyEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/IsEmptyEvaluator.java
new file mode 100644
index 0000000..e6e9fc9
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/IsEmptyEvaluator.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+
+public class IsEmptyEvaluator extends BooleanEvaluator {
+
+ private final Evaluator<?> subjectEvaluator;
+
+ public IsEmptyEvaluator(final Evaluator<?> subjectEvaluator) {
+ this.subjectEvaluator = subjectEvaluator;
+ }
+
+ @Override
+ public QueryResult<Boolean> evaluate(final Map<String, String> attributes) {
+ final Object subjectValue = subjectEvaluator.evaluate(attributes).getValue();
+ return new BooleanQueryResult(subjectValue == null || subjectValue.toString().trim().isEmpty());
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subjectEvaluator;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/IsNullEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/IsNullEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/IsNullEvaluator.java
new file mode 100644
index 0000000..a9d678c
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/IsNullEvaluator.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+
+public class IsNullEvaluator extends BooleanEvaluator {
+
+ private final Evaluator<?> subject;
+
+ public IsNullEvaluator(final Evaluator<?> subject) {
+ this.subject = subject;
+ }
+
+ @Override
+ public QueryResult<Boolean> evaluate(final Map<String, String> attributes) {
+ final Object subjectValue = subject.evaluate(attributes).getValue();
+ return new BooleanQueryResult(subjectValue == null);
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/LastIndexOfEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/LastIndexOfEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/LastIndexOfEvaluator.java
new file mode 100644
index 0000000..3ebbee7
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/LastIndexOfEvaluator.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.NumberEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.NumberQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+
+public class LastIndexOfEvaluator extends NumberEvaluator {
+
+ private final Evaluator<String> subject;
+ private final Evaluator<String> indexEvaluator;
+
+ public LastIndexOfEvaluator(final Evaluator<String> subject, final Evaluator<String> indexEvaluator) {
+ this.subject = subject;
+ this.indexEvaluator = indexEvaluator;
+ }
+
+ @Override
+ public QueryResult<Long> evaluate(final Map<String, String> attributes) {
+ final String subjectValue = subject.evaluate(attributes).getValue();
+ if (subjectValue == null) {
+ return new NumberQueryResult(-1L);
+ }
+ final String indexEvalValue = indexEvaluator.evaluate(attributes).getValue();
+
+ return new NumberQueryResult((long) subjectValue.lastIndexOf(indexEvalValue));
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/LengthEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/LengthEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/LengthEvaluator.java
new file mode 100644
index 0000000..46bc24c
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/LengthEvaluator.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.NumberEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.NumberQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+
+public class LengthEvaluator extends NumberEvaluator {
+
+ private final Evaluator<String> subject;
+
+ public LengthEvaluator(final Evaluator<String> subject) {
+ this.subject = subject;
+ }
+
+ @Override
+ public QueryResult<Long> evaluate(final Map<String, String> attributes) {
+ final String subjectValue = subject.evaluate(attributes).getValue();
+ return new NumberQueryResult((long) (subjectValue == null ? 0 : subjectValue.length()));
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
[26/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/util/PeerStatusCache.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/util/PeerStatusCache.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/util/PeerStatusCache.java
new file mode 100644
index 0000000..c52b4b7
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/util/PeerStatusCache.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.util;
+
+import java.util.Set;
+
+import org.apache.nifi.remote.PeerStatus;
+
+public class PeerStatusCache {
+
+ private final Set<PeerStatus> statuses;
+ private final long timestamp;
+
+ public PeerStatusCache(final Set<PeerStatus> statuses) {
+ this(statuses, System.currentTimeMillis());
+ }
+
+ public PeerStatusCache(final Set<PeerStatus> statuses, final long timestamp) {
+ this.statuses = statuses;
+ this.timestamp = timestamp;
+ }
+
+ public Set<PeerStatus> getStatuses() {
+ return statuses;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/util/StandardDataPacket.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/util/StandardDataPacket.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/util/StandardDataPacket.java
new file mode 100644
index 0000000..70bb324
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/util/StandardDataPacket.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.util;
+
+import java.io.InputStream;
+import java.util.Map;
+
+import org.apache.nifi.remote.protocol.DataPacket;
+import org.apache.nifi.stream.io.LimitingInputStream;
+import org.apache.nifi.stream.io.MinimumLengthInputStream;
+
+public class StandardDataPacket implements DataPacket {
+
+ private final Map<String, String> attributes;
+ private final InputStream stream;
+ private final long size;
+
+ public StandardDataPacket(final Map<String, String> attributes, final InputStream stream, final long size) {
+ this.attributes = attributes;
+ this.stream = new MinimumLengthInputStream(new LimitingInputStream(stream, size), size);
+ this.size = size;
+ }
+
+ public Map<String, String> getAttributes() {
+ return attributes;
+ }
+
+ public InputStream getData() {
+ return stream;
+ }
+
+ public long getSize() {
+ return size;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/test/java/org/apache/nifi/remote/client/socket/TestEndpointConnectionStatePool.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/test/java/org/apache/nifi/remote/client/socket/TestEndpointConnectionStatePool.java b/nifi-commons/nifi-site-to-site-client/src/test/java/org/apache/nifi/remote/client/socket/TestEndpointConnectionStatePool.java
new file mode 100644
index 0000000..8336745
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/test/java/org/apache/nifi/remote/client/socket/TestEndpointConnectionStatePool.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.client.socket;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.nifi.remote.PeerStatus;
+import org.apache.nifi.remote.TransferDirection;
+import org.apache.nifi.remote.cluster.ClusterNodeInformation;
+import org.apache.nifi.remote.cluster.NodeInformation;
+import org.junit.Test;
+
+public class TestEndpointConnectionStatePool {
+
+ @Test
+ public void testFormulateDestinationListForOutput() throws IOException {
+ final ClusterNodeInformation clusterNodeInfo = new ClusterNodeInformation();
+ final List<NodeInformation> collection = new ArrayList<>();
+ collection.add(new NodeInformation("ShouldGetMedium", 1, 1111, true, 4096));
+ collection.add(new NodeInformation("ShouldGetLots", 2, 2222, true, 10240));
+ collection.add(new NodeInformation("ShouldGetLittle", 3, 3333, true, 1024));
+ collection.add(new NodeInformation("ShouldGetMedium", 4, 4444, true, 4096));
+ collection.add(new NodeInformation("ShouldGetMedium", 5, 5555, true, 4096));
+
+ clusterNodeInfo.setNodeInformation(collection);
+ final List<PeerStatus> destinations = EndpointConnectionPool.formulateDestinationList(clusterNodeInfo, TransferDirection.RECEIVE);
+ for (final PeerStatus peerStatus : destinations) {
+ System.out.println(peerStatus.getPeerDescription());
+ }
+ }
+
+ @Test
+ public void testFormulateDestinationListForOutputHugeDifference() throws IOException {
+ final ClusterNodeInformation clusterNodeInfo = new ClusterNodeInformation();
+ final List<NodeInformation> collection = new ArrayList<>();
+ collection.add(new NodeInformation("ShouldGetLittle", 1, 1111, true, 500));
+ collection.add(new NodeInformation("ShouldGetLots", 2, 2222, true, 50000));
+
+ clusterNodeInfo.setNodeInformation(collection);
+ final List<PeerStatus> destinations = EndpointConnectionPool.formulateDestinationList(clusterNodeInfo, TransferDirection.RECEIVE);
+ for (final PeerStatus peerStatus : destinations) {
+ System.out.println(peerStatus.getPeerDescription());
+ }
+ }
+
+ @Test
+ public void testFormulateDestinationListForInputPorts() throws IOException {
+ final ClusterNodeInformation clusterNodeInfo = new ClusterNodeInformation();
+ final List<NodeInformation> collection = new ArrayList<>();
+ collection.add(new NodeInformation("ShouldGetMedium", 1, 1111, true, 4096));
+ collection.add(new NodeInformation("ShouldGetLittle", 2, 2222, true, 10240));
+ collection.add(new NodeInformation("ShouldGetLots", 3, 3333, true, 1024));
+ collection.add(new NodeInformation("ShouldGetMedium", 4, 4444, true, 4096));
+ collection.add(new NodeInformation("ShouldGetMedium", 5, 5555, true, 4096));
+
+ clusterNodeInfo.setNodeInformation(collection);
+ final List<PeerStatus> destinations = EndpointConnectionPool.formulateDestinationList(clusterNodeInfo, TransferDirection.SEND);
+ for (final PeerStatus peerStatus : destinations) {
+ System.out.println(peerStatus.getPeerDescription());
+ }
+ }
+
+ @Test
+ public void testFormulateDestinationListForInputPortsHugeDifference() throws IOException {
+ final ClusterNodeInformation clusterNodeInfo = new ClusterNodeInformation();
+ final List<NodeInformation> collection = new ArrayList<>();
+ collection.add(new NodeInformation("ShouldGetLots", 1, 1111, true, 500));
+ collection.add(new NodeInformation("ShouldGetLittle", 2, 2222, true, 50000));
+
+ clusterNodeInfo.setNodeInformation(collection);
+ final List<PeerStatus> destinations = EndpointConnectionPool.formulateDestinationList(clusterNodeInfo, TransferDirection.SEND);
+ for (final PeerStatus peerStatus : destinations) {
+ System.out.println(peerStatus.getPeerDescription());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/test/java/org/apache/nifi/remote/client/socket/TestSiteToSiteClient.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/test/java/org/apache/nifi/remote/client/socket/TestSiteToSiteClient.java b/nifi-commons/nifi-site-to-site-client/src/test/java/org/apache/nifi/remote/client/socket/TestSiteToSiteClient.java
new file mode 100644
index 0000000..155fc95
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/test/java/org/apache/nifi/remote/client/socket/TestSiteToSiteClient.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.client.socket;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.nifi.remote.Transaction;
+import org.apache.nifi.remote.TransferDirection;
+import org.apache.nifi.remote.client.SiteToSiteClient;
+import org.apache.nifi.remote.protocol.DataPacket;
+import org.apache.nifi.remote.util.StandardDataPacket;
+import org.apache.nifi.stream.io.StreamUtils;
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class TestSiteToSiteClient {
+
+ @Test
+ @Ignore("For local testing only; not really a unit test but a manual test")
+ public void testReceive() throws IOException {
+ System.setProperty("org.slf4j.simpleLogger.log.org.apache.nifi.remote", "DEBUG");
+
+ final SiteToSiteClient client = new SiteToSiteClient.Builder()
+ .url("http://localhost:8080/nifi")
+ .portName("cba")
+ .requestBatchCount(10)
+ .build();
+
+ try {
+ for (int i = 0; i < 1000; i++) {
+ final Transaction transaction = client.createTransaction(TransferDirection.RECEIVE);
+ Assert.assertNotNull(transaction);
+
+ DataPacket packet;
+ while (true) {
+ packet = transaction.receive();
+ if (packet == null) {
+ break;
+ }
+
+ final InputStream in = packet.getData();
+ final long size = packet.getSize();
+ final byte[] buff = new byte[(int) size];
+
+ StreamUtils.fillBuffer(in, buff);
+ }
+
+ transaction.confirm();
+ transaction.complete();
+ }
+ } finally {
+ client.close();
+ }
+ }
+
+ @Test
+ @Ignore("For local testing only; not really a unit test but a manual test")
+ public void testSend() throws IOException {
+ System.setProperty("org.slf4j.simpleLogger.log.org.apache.nifi.remote", "DEBUG");
+
+ final SiteToSiteClient client = new SiteToSiteClient.Builder()
+ .url("http://localhost:8080/nifi")
+ .portName("input")
+ .build();
+
+ try {
+ final Transaction transaction = client.createTransaction(TransferDirection.SEND);
+ Assert.assertNotNull(transaction);
+
+ final Map<String, String> attrs = new HashMap<>();
+ attrs.put("site-to-site", "yes, please!");
+ final byte[] bytes = "Hello".getBytes();
+ final ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
+ final DataPacket packet = new StandardDataPacket(attrs, bais, bytes.length);
+ transaction.send(packet);
+
+ transaction.confirm();
+ transaction.complete();
+ } finally {
+ client.close();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-socket-utils/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-socket-utils/pom.xml b/nifi-commons/nifi-socket-utils/pom.xml
new file mode 100644
index 0000000..4ed05ef
--- /dev/null
+++ b/nifi-commons/nifi-socket-utils/pom.xml
@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-commons</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>nifi-socket-utils</artifactId>
+ <description>Utilities for socket communication</description>
+ <dependencies>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-utils</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-logging-utils</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-net</groupId>
+ <artifactId>commons-net</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-properties</artifactId>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/nio/AbstractChannelReader.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/nio/AbstractChannelReader.java b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/nio/AbstractChannelReader.java
new file mode 100644
index 0000000..cc24575
--- /dev/null
+++ b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/nio/AbstractChannelReader.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.io.nio;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.SelectionKey;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.nifi.io.nio.consumer.StreamConsumer;
+import org.apache.nifi.io.nio.consumer.StreamConsumerFactory;
+
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class AbstractChannelReader implements Runnable {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(AbstractChannelReader.class);
+ private final String uniqueId;
+ private final SelectionKey key;
+ private final BufferPool bufferPool;
+ private final StreamConsumer consumer;
+ private final AtomicBoolean isClosed = new AtomicBoolean(false);
+ private final AtomicReference<ScheduledFuture<?>> future = new AtomicReference<>(null);//the future on which this reader runs...
+
+ public AbstractChannelReader(final String id, final SelectionKey key, final BufferPool empties, final StreamConsumerFactory consumerFactory) {
+ this.uniqueId = id;
+ this.key = key;
+ this.bufferPool = empties;
+ this.consumer = consumerFactory.newInstance(id);
+ consumer.setReturnBufferQueue(bufferPool);
+ }
+
+ protected void setScheduledFuture(final ScheduledFuture<?> future) {
+ this.future.set(future);
+ }
+
+ protected ScheduledFuture<?> getScheduledFuture() {
+ return future.get();
+ }
+
+ protected SelectionKey getSelectionKey() {
+ return key;
+ }
+
+ public boolean isClosed() {
+ return isClosed.get();
+ }
+
+ private void closeStream() {
+ if (isClosed.get()) {
+ return;
+ }
+ try {
+ isClosed.set(true);
+ future.get().cancel(false);
+ key.cancel();
+ key.channel().close();
+ } catch (final IOException ioe) {
+ LOGGER.warn("Unable to cleanly close stream due to " + ioe);
+ } finally {
+ consumer.signalEndOfStream();
+ }
+ }
+
+ /**
+ * Allows a subclass to specifically handle how it reads from the given
+ * key's channel into the given buffer.
+ *
+ * @param key of channel to read from
+ * @param buffer to fill
+ * @return the number of bytes read in the final read cycle. A value of zero
+ * or more indicates the channel is still open but a value of -1 indicates
+ * end of stream.
+ * @throws IOException if reading from channel causes failure
+ */
+ protected abstract int fillBuffer(SelectionKey key, ByteBuffer buffer) throws IOException;
+
+ @Override
+ public final void run() {
+ if (!key.isValid() || consumer.isConsumerFinished()) {
+ closeStream();
+ return;
+ }
+ if (!key.isReadable()) {
+ return;//there is nothing available to read...or we aren't allow to read due to throttling
+ }
+ ByteBuffer buffer = null;
+ try {
+ buffer = bufferPool.poll();
+ if (buffer == null) {
+ return; // no buffers available - come back later
+ }
+ final int bytesRead = fillBuffer(key, buffer);
+ buffer.flip();
+ if (buffer.remaining() > 0) {
+ consumer.addFilledBuffer(buffer);
+ buffer = null; //clear the reference - is now the consumer's responsiblity
+ } else {
+ buffer.clear();
+ bufferPool.returnBuffer(buffer, 0);
+ buffer = null; //clear the reference - is now back to the queue
+ }
+ if (bytesRead < 0) { //we've reached the end
+ closeStream();
+ }
+ } catch (final Exception ioe) {
+ closeStream();
+ LOGGER.error("Closed channel reader " + this + " due to " + ioe);
+ } finally {
+ if (buffer != null) {
+ buffer.clear();
+ bufferPool.returnBuffer(buffer, 0);
+ }
+ }
+ }
+
+ @Override
+ public final boolean equals(final Object obj) {
+ if (obj == null) {
+ return false;
+ }
+ if (obj == this) {
+ return true;
+ }
+ if (obj.getClass() != getClass()) {
+ return false;
+ }
+ AbstractChannelReader rhs = (AbstractChannelReader) obj;
+ return new EqualsBuilder().appendSuper(super.equals(obj)).append(uniqueId, rhs.uniqueId).isEquals();
+ }
+
+ @Override
+ public final int hashCode() {
+ return new HashCodeBuilder(17, 37).append(uniqueId).toHashCode();
+ }
+
+ @Override
+ public final String toString() {
+ return new ToStringBuilder(this, ToStringStyle.NO_FIELD_NAMES_STYLE).append(uniqueId).toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/nio/BufferPool.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/nio/BufferPool.java b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/nio/BufferPool.java
new file mode 100644
index 0000000..007034b
--- /dev/null
+++ b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/nio/BufferPool.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.io.nio;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingDeque;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class BufferPool implements Runnable {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(BufferPool.class);
+ final BlockingQueue<ByteBuffer> bufferPool;
+ private final static double ONE_MB = 1 << 20;
+ private Calendar lastRateSampleTime = Calendar.getInstance();
+ private final Calendar startTime = Calendar.getInstance();
+ double lastRateSampleMBps = -1.0;
+ double overallMBps = -1.0;
+ private long totalBytesExtracted = 0L;
+ private long lastTotalBytesExtracted = 0L;
+ final double maxRateMBps;
+
+ public BufferPool(final int bufferCount, final int bufferCapacity, final boolean allocateDirect, final double maxRateMBps) {
+ bufferPool = new LinkedBlockingDeque<>(BufferPool.createBuffers(bufferCount, bufferCapacity, allocateDirect));
+ this.maxRateMBps = maxRateMBps;
+ }
+
+ /**
+ * Returns the given buffer to the pool - and clears it.
+ *
+ * @param buffer buffer to return
+ * @param bytesProcessed bytes processed for this buffer being returned
+ * @return true if buffer returned to pool
+ */
+ public synchronized boolean returnBuffer(ByteBuffer buffer, final int bytesProcessed) {
+ totalBytesExtracted += bytesProcessed;
+ buffer.clear();
+ return bufferPool.add(buffer);
+ }
+
+ //here we enforce the desired rate we want by restricting access to buffers when we're over rate
+ public synchronized ByteBuffer poll() {
+ computeRate();
+ final double weightedAvg = (lastRateSampleMBps * 0.7) + (overallMBps * 0.3);
+ if (overallMBps >= maxRateMBps || weightedAvg >= maxRateMBps) {
+ return null;
+ }
+ return bufferPool.poll();
+ }
+
+ public int size() {
+ return bufferPool.size();
+ }
+
+ private synchronized void computeRate() {
+ final Calendar now = Calendar.getInstance();
+ final long measurementDurationMillis = now.getTimeInMillis() - lastRateSampleTime.getTimeInMillis();
+ final double duractionSecs = ((double) measurementDurationMillis) / 1000.0;
+ if (duractionSecs >= 0.75) { //recompute every 3/4 second or when we're too fast
+ final long totalDuractionMillis = now.getTimeInMillis() - startTime.getTimeInMillis();
+ final double totalDurationSecs = ((double) totalDuractionMillis) / 1000.0;
+ final long differenceBytes = totalBytesExtracted - lastTotalBytesExtracted;
+ lastTotalBytesExtracted = totalBytesExtracted;
+ lastRateSampleTime = now;
+ final double bps = ((double) differenceBytes) / duractionSecs;
+ final double totalBps = ((double) totalBytesExtracted / totalDurationSecs);
+ lastRateSampleMBps = bps / ONE_MB;
+ overallMBps = totalBps / ONE_MB;
+ }
+ }
+
+ public static List<ByteBuffer> createBuffers(final int bufferCount, final int bufferCapacity, final boolean allocateDirect) {
+ final List<ByteBuffer> buffers = new ArrayList<>();
+ for (int i = 0; i < bufferCount; i++) {
+ final ByteBuffer buffer = (allocateDirect) ? ByteBuffer.allocateDirect(bufferCapacity) : ByteBuffer.allocate(bufferCapacity);
+ buffers.add(buffer);
+ }
+ return buffers;
+ }
+
+ private void logChannelReadRates() {
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug(String.format("Overall rate= %,.4f MB/s / Current Rate= %,.4f MB/s / Total Bytes Read= %d", overallMBps, lastRateSampleMBps, totalBytesExtracted));
+ }
+ }
+
+ @Override
+ public void run() {
+ computeRate();
+ logChannelReadRates();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/nio/ChannelDispatcher.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/nio/ChannelDispatcher.java b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/nio/ChannelDispatcher.java
new file mode 100644
index 0000000..a4308e3
--- /dev/null
+++ b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/nio/ChannelDispatcher.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.io.nio;
+
+import java.io.IOException;
+import java.nio.channels.DatagramChannel;
+import java.nio.channels.SelectableChannel;
+import java.nio.channels.SelectionKey;
+import java.nio.channels.Selector;
+import java.nio.channels.ServerSocketChannel;
+import java.nio.channels.SocketChannel;
+import java.util.Iterator;
+import java.util.UUID;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.nifi.io.nio.consumer.StreamConsumerFactory;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public final class ChannelDispatcher implements Runnable {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(ChannelDispatcher.class);
+ private final Selector serverSocketSelector;
+ private final Selector socketChannelSelector;
+ private final ScheduledExecutorService executor;
+ private final BufferPool emptyBuffers;
+ private final StreamConsumerFactory factory;
+ private final AtomicLong channelReaderFrequencyMilliseconds = new AtomicLong(DEFAULT_CHANNEL_READER_PERIOD_MILLISECONDS);
+ private final long timeout;
+ private final boolean readSingleDatagram;
+ private volatile boolean stop = false;
+ public static final long DEFAULT_CHANNEL_READER_PERIOD_MILLISECONDS = 100L;
+
+ public ChannelDispatcher(final Selector serverSocketSelector, final Selector socketChannelSelector, final ScheduledExecutorService service,
+ final StreamConsumerFactory factory, final BufferPool buffers, final long timeout, final TimeUnit unit, final boolean readSingleDatagram) {
+ this.serverSocketSelector = serverSocketSelector;
+ this.socketChannelSelector = socketChannelSelector;
+ this.executor = service;
+ this.factory = factory;
+ emptyBuffers = buffers;
+ this.timeout = TimeUnit.MILLISECONDS.convert(timeout, unit);
+ this.readSingleDatagram = readSingleDatagram;
+ }
+
+ public void setChannelReaderFrequency(final long period, final TimeUnit timeUnit) {
+ channelReaderFrequencyMilliseconds.set(TimeUnit.MILLISECONDS.convert(period, timeUnit));
+ }
+
+ @Override
+ public void run() {
+ while (!stop) {
+ try {
+ selectServerSocketKeys();
+ selectSocketChannelKeys();
+ } catch (final Exception ex) {
+ LOGGER.warn("Key selection failed: {} Normal during shutdown.", new Object[]{ex});
+ }
+ }
+ }
+
+ /*
+ * When serverSocketsChannels are registered with the selector, want each invoke of this method to loop through all
+ * channels' keys.
+ *
+ * @throws IOException if unable to select keys
+ */
+ private void selectServerSocketKeys() throws IOException {
+ int numSelected = serverSocketSelector.select(timeout);
+ if (numSelected == 0) {
+ return;
+ }
+
+ // for each registered server socket - see if any connections are waiting to be established
+ final Iterator<SelectionKey> itr = serverSocketSelector.selectedKeys().iterator();
+ while (itr.hasNext()) {
+ SelectionKey serverSocketkey = itr.next();
+ final SelectableChannel channel = serverSocketkey.channel();
+ AbstractChannelReader reader = null;
+ if (serverSocketkey.isValid() && serverSocketkey.isAcceptable()) {
+ final ServerSocketChannel ssChannel = (ServerSocketChannel) serverSocketkey.channel();
+ final SocketChannel sChannel = ssChannel.accept();
+ if (sChannel != null) {
+ sChannel.configureBlocking(false);
+ final SelectionKey socketChannelKey = sChannel.register(socketChannelSelector, SelectionKey.OP_READ);
+ final String readerId = sChannel.socket().toString();
+ reader = new SocketChannelReader(readerId, socketChannelKey, emptyBuffers, factory);
+ final ScheduledFuture<?> readerFuture = executor.scheduleWithFixedDelay(reader, 10L,
+ channelReaderFrequencyMilliseconds.get(), TimeUnit.MILLISECONDS);
+ reader.setScheduledFuture(readerFuture);
+ socketChannelKey.attach(reader);
+ }
+ }
+ itr.remove(); // do this so that the next select operation returns a positive value; otherwise, it will return 0.
+ if (reader != null && LOGGER.isDebugEnabled()) {
+ LOGGER.debug(this + " New Connection established. Server channel: " + channel + " Reader: " + reader);
+ }
+ }
+ }
+
+ /*
+ * When invoking this method, only want to iterate through the selected keys once. When a key is entered into the selectors
+ * selected key set, select will return a positive value. The next select will return 0 if nothing has changed. Note that
+ * the selected key set is not manually changed via a remove operation.
+ *
+ * @throws IOException if unable to select keys
+ */
+ private void selectSocketChannelKeys() throws IOException {
+ // once a channel associated with a key in this selector is 'ready', it causes this select to immediately return.
+ // thus, for each trip through the run() we only get hit with one real timeout...the one in selectServerSocketKeys.
+ int numSelected = socketChannelSelector.select(timeout);
+ if (numSelected == 0) {
+ return;
+ }
+
+ for (SelectionKey socketChannelKey : socketChannelSelector.selectedKeys()) {
+ final SelectableChannel channel = socketChannelKey.channel();
+ AbstractChannelReader reader = null;
+ // there are 2 kinds of channels in this selector, both which have their own readers and are executed in their own
+ // threads. We will get here whenever a new SocketChannel is created due to an incoming connection. However,
+ // for a DatagramChannel we don't want to create a new reader unless it is a new DatagramChannel. The only
+ // way to tell if it's new is the lack of an attachment.
+ if (channel instanceof DatagramChannel && socketChannelKey.attachment() == null) {
+ reader = new DatagramChannelReader(UUID.randomUUID().toString(), socketChannelKey, emptyBuffers, factory, readSingleDatagram);
+ socketChannelKey.attach(reader);
+ final ScheduledFuture<?> readerFuture = executor.scheduleWithFixedDelay(reader, 10L, channelReaderFrequencyMilliseconds.get(),
+ TimeUnit.MILLISECONDS);
+ reader.setScheduledFuture(readerFuture);
+ }
+ if (reader != null && LOGGER.isDebugEnabled()) {
+ LOGGER.debug(this + " New Connection established. Server channel: " + channel + " Reader: " + reader);
+ }
+ }
+
+ }
+
+ public void stop() {
+ stop = true;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/nio/ChannelListener.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/nio/ChannelListener.java b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/nio/ChannelListener.java
new file mode 100644
index 0000000..ab77063
--- /dev/null
+++ b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/nio/ChannelListener.java
@@ -0,0 +1,227 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.io.nio;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.StandardSocketOptions;
+import java.nio.channels.DatagramChannel;
+import java.nio.channels.SelectionKey;
+import java.nio.channels.Selector;
+import java.nio.channels.ServerSocketChannel;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.nifi.io.nio.consumer.StreamConsumerFactory;
+
+import org.apache.commons.io.IOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class provides the entry point to NIO based socket listeners for NiFi
+ * processors and services. There are 2 supported types of Listeners, Datagram
+ * (UDP based transmissions) and ServerSocket (TCP based transmissions). This
+ * will create the ChannelDispatcher, which is a Runnable and is controlled via
+ * the ScheduledExecutorService, which is also created by this class. The
+ * ChannelDispatcher handles connections to the ServerSocketChannels and creates
+ * the readers associated with the resulting SocketChannels. Additionally, this
+ * creates and manages two Selectors, one for ServerSocketChannels and another
+ * for SocketChannels and DatagramChannels.
+ *
+ * The threading model for this consists of one thread for the
+ * ChannelDispatcher, one thread per added SocketChannel reader, one thread per
+ * added DatagramChannel reader. The ChannelDispatcher is not scheduled with
+ * fixed delay as the others are. It is throttled by the provided timeout value.
+ * Within the ChannelDispatcher there are two blocking operations which will
+ * block for the given timeout each time through the enclosing loop.
+ *
+ * All channels are cached in one of the two Selectors via their SelectionKey.
+ * The serverSocketSelector maintains all the added ServerSocketChannels; the
+ * socketChannelSelector maintains the all the add DatagramChannels and the
+ * created SocketChannels. Further, the SelectionKey of the DatagramChannel and
+ * the SocketChannel is injected with the channel's associated reader.
+ *
+ * All ChannelReaders will get throttled by the unavailability of buffers in the
+ * provided BufferPool. This is designed to create back pressure.
+ *
+ */
+public final class ChannelListener {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(ChannelListener.class);
+ private final ScheduledExecutorService executor;
+ private final Selector serverSocketSelector; // used to listen for new connections
+ private final Selector socketChannelSelector; // used to listen on existing connections
+ private final ChannelDispatcher channelDispatcher;
+ private final BufferPool bufferPool;
+ private final int initialBufferPoolSize;
+ private volatile long channelReaderFrequencyMSecs = 50;
+
+ public ChannelListener(final int threadPoolSize, final StreamConsumerFactory consumerFactory, final BufferPool bufferPool, int timeout,
+ TimeUnit unit, final boolean readSingleDatagram) throws IOException {
+ this.executor = Executors.newScheduledThreadPool(threadPoolSize + 1); // need to allow for long running ChannelDispatcher thread
+ this.serverSocketSelector = Selector.open();
+ this.socketChannelSelector = Selector.open();
+ this.bufferPool = bufferPool;
+ this.initialBufferPoolSize = bufferPool.size();
+ channelDispatcher = new ChannelDispatcher(serverSocketSelector, socketChannelSelector, executor, consumerFactory, bufferPool,
+ timeout, unit, readSingleDatagram);
+ executor.schedule(channelDispatcher, 50, TimeUnit.MILLISECONDS);
+ }
+
+ public void setChannelReaderSchedulingPeriod(final long period, final TimeUnit unit) {
+ channelReaderFrequencyMSecs = TimeUnit.MILLISECONDS.convert(period, unit);
+ channelDispatcher.setChannelReaderFrequency(period, unit);
+ }
+
+ /**
+ * Adds a server socket channel for listening to connections.
+ *
+ * @param nicIPAddress - if null binds to wildcard address
+ * @param port - port to bind to
+ * @param receiveBufferSize - size of OS receive buffer to request. If less
+ * than 0 then will not be set and OS default will win.
+ * @throws IOException if unable to add socket
+ */
+ public void addServerSocket(final InetAddress nicIPAddress, final int port, final int receiveBufferSize)
+ throws IOException {
+ final ServerSocketChannel ssChannel = ServerSocketChannel.open();
+ ssChannel.configureBlocking(false);
+ if (receiveBufferSize > 0) {
+ ssChannel.setOption(StandardSocketOptions.SO_RCVBUF, receiveBufferSize);
+ final int actualReceiveBufSize = ssChannel.getOption(StandardSocketOptions.SO_RCVBUF);
+ if (actualReceiveBufSize < receiveBufferSize) {
+ LOGGER.warn(this + " attempted to set TCP Receive Buffer Size to "
+ + receiveBufferSize + " bytes but could only set to " + actualReceiveBufSize
+ + "bytes. You may want to consider changing the Operating System's "
+ + "maximum receive buffer");
+ }
+ }
+ ssChannel.setOption(StandardSocketOptions.SO_REUSEADDR, true);
+ ssChannel.bind(new InetSocketAddress(nicIPAddress, port));
+ ssChannel.register(serverSocketSelector, SelectionKey.OP_ACCEPT);
+ }
+
+ /**
+ * Binds to listen for data grams on the given local IPAddress/port
+ *
+ * @param nicIPAddress - if null will listen on wildcard address, which
+ * means datagrams will be received on all local network interfaces.
+ * Otherwise, will bind to the provided IP address associated with some NIC.
+ * @param port - the port to listen on
+ * @param receiveBufferSize - the number of bytes to request for a receive
+ * buffer from OS
+ * @throws IOException if unable to add channel
+ */
+ public void addDatagramChannel(final InetAddress nicIPAddress, final int port, final int receiveBufferSize)
+ throws IOException {
+ final DatagramChannel dChannel = createAndBindDatagramChannel(nicIPAddress, port, receiveBufferSize);
+ dChannel.register(socketChannelSelector, SelectionKey.OP_READ);
+ }
+
+ /**
+ * Binds to listen for data grams on the given local IPAddress/port and
+ * restricts receipt of datagrams to those from the provided host and port,
+ * must specify both. This improves performance for datagrams coming from a
+ * sender that is known a-priori.
+ *
+ * @param nicIPAddress - if null will listen on wildcard address, which
+ * means datagrams will be received on all local network interfaces.
+ * Otherwise, will bind to the provided IP address associated with some NIC.
+ * @param port - the port to listen on. This is used to provide a well-known
+ * destination for a sender.
+ * @param receiveBufferSize - the number of bytes to request for a receive
+ * buffer from OS
+ * @param sendingHost - the hostname, or IP address, of the sender of
+ * datagrams. Only datagrams from this host will be received. If this is
+ * null the wildcard ip is used, which means datagrams may be received from
+ * any network interface on the local host.
+ * @param sendingPort - the port used by the sender of datagrams. Only
+ * datagrams from this port will be received.
+ * @throws IOException if unable to add channel
+ */
+ public void addDatagramChannel(final InetAddress nicIPAddress, final int port, final int receiveBufferSize, final String sendingHost,
+ final Integer sendingPort) throws IOException {
+
+ if (sendingHost == null || sendingPort == null) {
+ addDatagramChannel(nicIPAddress, port, receiveBufferSize);
+ return;
+ }
+ final DatagramChannel dChannel = createAndBindDatagramChannel(nicIPAddress, port, receiveBufferSize);
+ dChannel.connect(new InetSocketAddress(sendingHost, sendingPort));
+ dChannel.register(socketChannelSelector, SelectionKey.OP_READ);
+ }
+
+ private DatagramChannel createAndBindDatagramChannel(final InetAddress nicIPAddress, final int port, final int receiveBufferSize)
+ throws IOException {
+ final DatagramChannel dChannel = DatagramChannel.open();
+ dChannel.configureBlocking(false);
+ if (receiveBufferSize > 0) {
+ dChannel.setOption(StandardSocketOptions.SO_RCVBUF, receiveBufferSize);
+ final int actualReceiveBufSize = dChannel.getOption(StandardSocketOptions.SO_RCVBUF);
+ if (actualReceiveBufSize < receiveBufferSize) {
+ LOGGER.warn(this + " attempted to set UDP Receive Buffer Size to "
+ + receiveBufferSize + " bytes but could only set to " + actualReceiveBufSize
+ + "bytes. You may want to consider changing the Operating System's "
+ + "maximum receive buffer");
+ }
+ }
+ dChannel.setOption(StandardSocketOptions.SO_REUSEADDR, true);
+ dChannel.bind(new InetSocketAddress(nicIPAddress, port));
+ return dChannel;
+ }
+
+ public void shutdown(final long period, final TimeUnit timeUnit) {
+ channelDispatcher.stop();
+ for (SelectionKey selectionKey : socketChannelSelector.keys()) {
+ final AbstractChannelReader reader = (AbstractChannelReader) selectionKey.attachment();
+ selectionKey.cancel();
+ if (reader != null) {
+ while (!reader.isClosed()) {
+ try {
+ Thread.sleep(channelReaderFrequencyMSecs);
+ } catch (InterruptedException e) {
+ }
+ }
+ final ScheduledFuture<?> readerFuture = reader.getScheduledFuture();
+ readerFuture.cancel(false);
+ }
+ IOUtils.closeQuietly(selectionKey.channel()); // should already be closed via reader, but if reader did not exist...
+ }
+ IOUtils.closeQuietly(socketChannelSelector);
+
+ for (SelectionKey selectionKey : serverSocketSelector.keys()) {
+ selectionKey.cancel();
+ IOUtils.closeQuietly(selectionKey.channel());
+ }
+ IOUtils.closeQuietly(serverSocketSelector);
+ executor.shutdown();
+ try {
+ executor.awaitTermination(period, timeUnit);
+ } catch (final InterruptedException ex) {
+ LOGGER.warn("Interrupted while trying to shutdown executor");
+ }
+ final int currentBufferPoolSize = bufferPool.size();
+ final String warning = (currentBufferPoolSize != initialBufferPoolSize) ? "Initial buffer count=" + initialBufferPoolSize
+ + " Current buffer count=" + currentBufferPoolSize
+ + " Could indicate a buffer leak. Ensure all consumers are executed until they complete." : "";
+ LOGGER.info("Channel listener shutdown. " + warning);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/nio/DatagramChannelReader.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/nio/DatagramChannelReader.java b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/nio/DatagramChannelReader.java
new file mode 100644
index 0000000..a4670b9
--- /dev/null
+++ b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/nio/DatagramChannelReader.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.io.nio;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.DatagramChannel;
+import java.nio.channels.SelectionKey;
+
+import org.apache.nifi.io.nio.consumer.StreamConsumerFactory;
+
+public final class DatagramChannelReader extends AbstractChannelReader {
+
+ public static final int MAX_UDP_PACKET_SIZE = 65507;
+
+ private final boolean readSingleDatagram;
+
+ public DatagramChannelReader(final String id, final SelectionKey key, final BufferPool empties, final StreamConsumerFactory consumerFactory,
+ final boolean readSingleDatagram) {
+ super(id, key, empties, consumerFactory);
+ this.readSingleDatagram = readSingleDatagram;
+ }
+
+ /**
+ * Will receive UDP data from channel and won't receive anything unless the
+ * given buffer has enough space for at least one full max udp packet.
+ *
+ * @param key selection key
+ * @param buffer to fill
+ * @return bytes read
+ * @throws IOException if error filling buffer from channel
+ */
+ @Override
+ protected int fillBuffer(final SelectionKey key, final ByteBuffer buffer) throws IOException {
+ final DatagramChannel dChannel = (DatagramChannel) key.channel();
+ final int initialBufferPosition = buffer.position();
+ while (buffer.remaining() > MAX_UDP_PACKET_SIZE && key.isValid() && key.isReadable()) {
+ if (dChannel.receive(buffer) == null || readSingleDatagram) {
+ break;
+ }
+ }
+ return buffer.position() - initialBufferPosition;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/nio/SocketChannelReader.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/nio/SocketChannelReader.java b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/nio/SocketChannelReader.java
new file mode 100644
index 0000000..29c2973
--- /dev/null
+++ b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/nio/SocketChannelReader.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.io.nio;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.SelectionKey;
+import java.nio.channels.SocketChannel;
+import org.apache.nifi.io.nio.consumer.StreamConsumerFactory;
+
+public final class SocketChannelReader extends AbstractChannelReader {
+
+ public SocketChannelReader(final String id, final SelectionKey key, final BufferPool empties, final StreamConsumerFactory consumerFactory) {
+ super(id, key, empties, consumerFactory);
+ }
+
+ /**
+ * Receives TCP data from the socket channel for the given key.
+ *
+ * @param key selection key
+ * @param buffer byte buffer to fill
+ * @return bytes read
+ * @throws IOException if error reading bytes
+ */
+ @Override
+ protected int fillBuffer(final SelectionKey key, final ByteBuffer buffer) throws IOException {
+ int bytesRead = 0;
+ final SocketChannel sChannel = (SocketChannel) key.channel();
+ while (key.isValid() && key.isReadable()) {
+ bytesRead = sChannel.read(buffer);
+ if (bytesRead <= 0) {
+ break;
+ }
+ }
+ return bytesRead;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/nio/consumer/AbstractStreamConsumer.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/nio/consumer/AbstractStreamConsumer.java b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/nio/consumer/AbstractStreamConsumer.java
new file mode 100644
index 0000000..bb57e26
--- /dev/null
+++ b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/nio/consumer/AbstractStreamConsumer.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.io.nio.consumer;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.nifi.io.nio.BufferPool;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
+
+/**
+ *
+ */
+public abstract class AbstractStreamConsumer implements StreamConsumer {
+
+ private final String uniqueId;
+ private BufferPool bufferPool = null;
+ private final BlockingQueue<ByteBuffer> filledBuffers = new LinkedBlockingQueue<>();
+ private final AtomicBoolean streamEnded = new AtomicBoolean(false);
+ private final AtomicBoolean consumerEnded = new AtomicBoolean(false);
+
+ public AbstractStreamConsumer(final String id) {
+ uniqueId = id;
+ }
+
+ @Override
+ public final void setReturnBufferQueue(final BufferPool returnQueue) {
+ bufferPool = returnQueue;
+ }
+
+ @Override
+ public final void addFilledBuffer(final ByteBuffer buffer) {
+ if (isConsumerFinished()) {
+ buffer.clear();
+ bufferPool.returnBuffer(buffer, buffer.remaining());
+ } else {
+ filledBuffers.add(buffer);
+ }
+ }
+
+ @Override
+ public final void process() throws IOException {
+ if (isConsumerFinished()) {
+ return;
+ }
+ if (streamEnded.get() && filledBuffers.isEmpty()) {
+ consumerEnded.set(true);
+ onConsumerDone();
+ return;
+ }
+ final ByteBuffer buffer = filledBuffers.poll();
+ if (buffer != null) {
+ final int bytesToProcess = buffer.remaining();
+ try {
+ processBuffer(buffer);
+ } finally {
+ buffer.clear();
+ bufferPool.returnBuffer(buffer, bytesToProcess);
+ }
+ }
+ }
+
+ protected abstract void processBuffer(ByteBuffer buffer) throws IOException;
+
+ @Override
+ public final void signalEndOfStream() {
+ streamEnded.set(true);
+ }
+
+ /**
+ * Convenience method that is called when the consumer is done processing
+ * based on being told the signal is end of stream and has processed all
+ * given buffers.
+ */
+ protected void onConsumerDone() {
+ }
+
+ @Override
+ public final boolean isConsumerFinished() {
+ return consumerEnded.get();
+ }
+
+ @Override
+ public final String getId() {
+ return uniqueId;
+ }
+
+ @Override
+ public final boolean equals(final Object obj) {
+ if (obj == null) {
+ return false;
+ }
+ if (obj == this) {
+ return true;
+ }
+ if (obj.getClass() != getClass()) {
+ return false;
+ }
+ AbstractStreamConsumer rhs = (AbstractStreamConsumer) obj;
+ return new EqualsBuilder().appendSuper(super.equals(obj)).append(uniqueId, rhs.uniqueId).isEquals();
+ }
+
+ @Override
+ public final int hashCode() {
+ return new HashCodeBuilder(19, 23).append(uniqueId).toHashCode();
+ }
+
+ @Override
+ public final String toString() {
+ return new ToStringBuilder(this, ToStringStyle.NO_FIELD_NAMES_STYLE).append(uniqueId).toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/nio/consumer/StreamConsumer.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/nio/consumer/StreamConsumer.java b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/nio/consumer/StreamConsumer.java
new file mode 100644
index 0000000..cac8d8b
--- /dev/null
+++ b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/nio/consumer/StreamConsumer.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.io.nio.consumer;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import org.apache.nifi.io.nio.BufferPool;
+
+/**
+ * A StreamConsumer must be thread safe. It may be accessed concurrently by a
+ * thread providing data to process and another thread that is processing that
+ * data.
+ *
+ */
+public interface StreamConsumer {
+
+ /**
+ * Will be called once just after construction. It provides the queue to
+ * which processed and emptied and cleared buffers must be returned. For
+ * each time <code>addFilledBuffer</code> is called there should be an
+ * associated add to this given queue. If not, buffers will run out and all
+ * stream processing will halt. READ THIS!!!
+ *
+ * @param returnQueue pool of buffers to use
+ */
+ void setReturnBufferQueue(BufferPool returnQueue);
+
+ /**
+ * Will be called by the thread that produces byte buffers with available
+ * data to be processed. If the consumer is finished this should simply
+ * return the given buffer to the return buffer queue (after it is cleared)
+ *
+ * @param buffer filled buffer
+ */
+ void addFilledBuffer(ByteBuffer buffer);
+
+ /**
+ * Will be called by the thread that executes the consumption of data. May
+ * be called many times though once <code>isConsumerFinished</code> returns
+ * true this method will likely do nothing.
+ *
+ * @throws java.io.IOException if there is an issue processing
+ */
+ void process() throws IOException;
+
+ /**
+ * Called once the end of the input stream is detected
+ */
+ void signalEndOfStream();
+
+ /**
+ * If true signals the consumer is done consuming data and will not process
+ * any more buffers.
+ *
+ * @return true if finished
+ */
+ boolean isConsumerFinished();
+
+ /**
+ * Uniquely identifies the consumer
+ *
+ * @return identifier of consumer
+ */
+ String getId();
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/nio/consumer/StreamConsumerFactory.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/nio/consumer/StreamConsumerFactory.java b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/nio/consumer/StreamConsumerFactory.java
new file mode 100644
index 0000000..ba858b6
--- /dev/null
+++ b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/nio/consumer/StreamConsumerFactory.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.io.nio.consumer;
+
+/**
+ *
+ */
+public interface StreamConsumerFactory {
+
+ StreamConsumer newInstance(String streamId);
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/SSLContextFactory.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/SSLContextFactory.java b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/SSLContextFactory.java
new file mode 100644
index 0000000..9c6cb82
--- /dev/null
+++ b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/SSLContextFactory.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.io.socket;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.security.KeyManagementException;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.SecureRandom;
+import java.security.UnrecoverableKeyException;
+import java.security.cert.CertificateException;
+
+import javax.net.ssl.KeyManager;
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.TrustManagerFactory;
+
+import org.apache.nifi.util.NiFiProperties;
+
+public class SSLContextFactory {
+
+ private final String keystore;
+ private final char[] keystorePass;
+ private final String keystoreType;
+ private final String truststore;
+ private final char[] truststorePass;
+ private final String truststoreType;
+
+ private final KeyManager[] keyManagers;
+ private final TrustManager[] trustManagers;
+
+ public SSLContextFactory(final NiFiProperties properties) throws NoSuchAlgorithmException, CertificateException, FileNotFoundException, IOException, KeyStoreException, UnrecoverableKeyException {
+ keystore = properties.getProperty(NiFiProperties.SECURITY_KEYSTORE);
+ keystorePass = getPass(properties.getProperty(NiFiProperties.SECURITY_KEYSTORE_PASSWD));
+ keystoreType = properties.getProperty(NiFiProperties.SECURITY_KEYSTORE_TYPE);
+
+ truststore = properties.getProperty(NiFiProperties.SECURITY_TRUSTSTORE);
+ truststorePass = getPass(properties.getProperty(NiFiProperties.SECURITY_TRUSTSTORE_PASSWD));
+ truststoreType = properties.getProperty(NiFiProperties.SECURITY_TRUSTSTORE_TYPE);
+
+ // prepare the keystore
+ final KeyStore keyStore = KeyStore.getInstance(keystoreType);
+ keyStore.load(new FileInputStream(keystore), keystorePass);
+ final KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
+ keyManagerFactory.init(keyStore, keystorePass);
+
+ // prepare the truststore
+ final KeyStore trustStore = KeyStore.getInstance(truststoreType);
+ trustStore.load(new FileInputStream(truststore), truststorePass);
+ final TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
+ trustManagerFactory.init(trustStore);
+
+ keyManagers = keyManagerFactory.getKeyManagers();
+ trustManagers = trustManagerFactory.getTrustManagers();
+ }
+
+ private static char[] getPass(final String password) {
+ return password == null ? null : password.toCharArray();
+ }
+
+ /**
+ * Creates a SSLContext instance using the given information.
+ *
+ *
+ * @return a SSLContext instance
+ * @throws java.security.KeyStoreException if problem with keystore
+ * @throws java.io.IOException if unable to create context
+ * @throws java.security.NoSuchAlgorithmException if algorithm isn't known
+ * @throws java.security.cert.CertificateException if certificate is invalid
+ * @throws java.security.UnrecoverableKeyException if the key cannot be recovered
+ * @throws java.security.KeyManagementException if the key is improper
+ */
+ public SSLContext createSslContext() throws KeyStoreException, IOException, NoSuchAlgorithmException, CertificateException,
+ UnrecoverableKeyException, KeyManagementException {
+
+ // initialize the ssl context
+ final SSLContext sslContext = SSLContext.getInstance("TLS");
+ sslContext.init(keyManagers, trustManagers, new SecureRandom());
+ sslContext.getDefaultSSLParameters().setNeedClientAuth(true);
+
+ return sslContext;
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/ServerSocketConfiguration.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/ServerSocketConfiguration.java b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/ServerSocketConfiguration.java
new file mode 100644
index 0000000..d6aca92
--- /dev/null
+++ b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/ServerSocketConfiguration.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.io.socket;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.security.KeyManagementException;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.UnrecoverableKeyException;
+import java.security.cert.CertificateException;
+
+import javax.net.ssl.SSLContext;
+
+public final class ServerSocketConfiguration {
+
+ private boolean needClientAuth;
+ private Integer socketTimeout;
+ private Boolean reuseAddress;
+ private Integer receiveBufferSize;
+ private SSLContextFactory sslContextFactory;
+
+ public ServerSocketConfiguration() {
+ }
+
+ public SSLContext createSSLContext()
+ throws KeyManagementException, NoSuchAlgorithmException, UnrecoverableKeyException, KeyStoreException, CertificateException, FileNotFoundException, IOException {
+ return sslContextFactory == null ? null : sslContextFactory.createSslContext();
+ }
+
+ public void setSSLContextFactory(final SSLContextFactory sslContextFactory) {
+ this.sslContextFactory = sslContextFactory;
+ }
+
+ public Integer getSocketTimeout() {
+ return socketTimeout;
+ }
+
+ public void setSocketTimeout(Integer socketTimeout) {
+ this.socketTimeout = socketTimeout;
+ }
+
+ public boolean getNeedClientAuth() {
+ return needClientAuth;
+ }
+
+ public void setNeedClientAuth(boolean needClientAuth) {
+ this.needClientAuth = needClientAuth;
+ }
+
+ public Boolean getReuseAddress() {
+ return reuseAddress;
+ }
+
+ public void setReuseAddress(Boolean reuseAddress) {
+ this.reuseAddress = reuseAddress;
+ }
+
+ public Integer getReceiveBufferSize() {
+ return receiveBufferSize;
+ }
+
+ public void setReceiveBufferSize(Integer receiveBufferSize) {
+ this.receiveBufferSize = receiveBufferSize;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/SocketConfiguration.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/SocketConfiguration.java b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/SocketConfiguration.java
new file mode 100644
index 0000000..8b803dc
--- /dev/null
+++ b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/SocketConfiguration.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.io.socket;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.security.KeyManagementException;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.UnrecoverableKeyException;
+import java.security.cert.CertificateException;
+
+import javax.net.ssl.SSLContext;
+
+public final class SocketConfiguration {
+
+ private Integer socketTimeout;
+ private Integer receiveBufferSize;
+ private Integer sendBufferSize;
+ private Boolean reuseAddress;
+ private Boolean keepAlive;
+ private Boolean oobInline;
+ private Boolean tcpNoDelay;
+ private Integer trafficClass;
+ private SSLContextFactory sslContextFactory;
+
+ public SSLContext createSSLContext()
+ throws KeyManagementException, NoSuchAlgorithmException, UnrecoverableKeyException, KeyStoreException, CertificateException, FileNotFoundException, IOException {
+ return sslContextFactory == null ? null : sslContextFactory.createSslContext();
+ }
+
+ public void setSSLContextFactory(final SSLContextFactory sslContextFactory) {
+ this.sslContextFactory = sslContextFactory;
+ }
+
+ public Integer getSocketTimeout() {
+ return socketTimeout;
+ }
+
+ public void setSocketTimeout(Integer socketTimeout) {
+ this.socketTimeout = socketTimeout;
+ }
+
+ public Boolean getReuseAddress() {
+ return reuseAddress;
+ }
+
+ public void setReuseAddress(Boolean reuseAddress) {
+ this.reuseAddress = reuseAddress;
+ }
+
+ public Boolean getKeepAlive() {
+ return keepAlive;
+ }
+
+ public void setKeepAlive(Boolean keepAlive) {
+ this.keepAlive = keepAlive;
+ }
+
+ public Boolean getOobInline() {
+ return oobInline;
+ }
+
+ public void setOobInline(Boolean oobInline) {
+ this.oobInline = oobInline;
+ }
+
+ public Integer getReceiveBufferSize() {
+ return receiveBufferSize;
+ }
+
+ public void setReceiveBufferSize(Integer receiveBufferSize) {
+ this.receiveBufferSize = receiveBufferSize;
+ }
+
+ public Integer getSendBufferSize() {
+ return sendBufferSize;
+ }
+
+ public void setSendBufferSize(Integer sendBufferSize) {
+ this.sendBufferSize = sendBufferSize;
+ }
+
+ public Boolean getTcpNoDelay() {
+ return tcpNoDelay;
+ }
+
+ public void setTcpNoDelay(Boolean tcpNoDelay) {
+ this.tcpNoDelay = tcpNoDelay;
+ }
+
+ public Integer getTrafficClass() {
+ return trafficClass;
+ }
+
+ public void setTrafficClass(Integer trafficClass) {
+ this.trafficClass = trafficClass;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/SocketListener.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/SocketListener.java b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/SocketListener.java
new file mode 100644
index 0000000..b509035
--- /dev/null
+++ b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/SocketListener.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.io.socket;
+
+import java.io.IOException;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.SocketException;
+import java.net.SocketTimeoutException;
+import java.security.KeyManagementException;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.UnrecoverableKeyException;
+import java.security.cert.CertificateException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.nifi.logging.NiFiLog;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Implements a listener for TCP/IP messages sent over unicast socket.
+ *
+ */
+public abstract class SocketListener {
+
+ private static final int DEFAULT_SHUTDOWN_LISTENER_SECONDS = 5;
+ private static final Logger logger = new NiFiLog(LoggerFactory.getLogger(SocketListener.class));
+ private volatile ExecutorService executorService; // volatile to guarantee most current value is visible
+ private volatile ServerSocket serverSocket; // volatile to guarantee most current value is visible
+ private final int numThreads;
+ private final int port;
+ private final ServerSocketConfiguration configuration;
+ private final AtomicInteger shutdownListenerSeconds = new AtomicInteger(DEFAULT_SHUTDOWN_LISTENER_SECONDS);
+
+ public SocketListener(
+ final int numThreads,
+ final int port,
+ final ServerSocketConfiguration configuration) {
+
+ if (numThreads <= 0) {
+ throw new IllegalArgumentException("Number of threads may not be less than or equal to zero.");
+ } else if (configuration == null) {
+ throw new IllegalArgumentException("Server socket configuration may not be null.");
+ }
+
+ this.numThreads = numThreads;
+ this.port = port;
+ this.configuration = configuration;
+ }
+
+ /**
+ * Implements the action to perform when a new socket request is received.
+ * This class will close the socket.
+ *
+ * @param socket the socket
+ */
+ public abstract void dispatchRequest(final Socket socket);
+
+ public void start() throws IOException {
+
+ if (isRunning()) {
+ return;
+ }
+
+ try {
+ serverSocket = SocketUtils.createServerSocket(port, configuration);
+ } catch (KeyManagementException | UnrecoverableKeyException | NoSuchAlgorithmException | KeyStoreException | CertificateException e) {
+ throw new IOException(e);
+ }
+
+ final ThreadFactory defaultThreadFactory = Executors.defaultThreadFactory();
+ executorService = Executors.newFixedThreadPool(numThreads, new ThreadFactory() {
+ private final AtomicLong threadCounter = new AtomicLong(0L);
+
+ @Override
+ public Thread newThread(final Runnable r) {
+ final Thread newThread = defaultThreadFactory.newThread(r);
+ newThread.setName("Process NCM Request-" + threadCounter.incrementAndGet());
+ return newThread;
+ }
+ });
+
+ final ExecutorService runnableExecServiceRef = executorService;
+ final ServerSocket runnableServerSocketRef = serverSocket;
+
+ final Thread t = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ while (runnableExecServiceRef.isShutdown() == false) {
+ Socket socket = null;
+ try {
+ try {
+ socket = runnableServerSocketRef.accept();
+ if (configuration.getSocketTimeout() != null) {
+ socket.setSoTimeout(configuration.getSocketTimeout());
+ }
+ } catch (final SocketTimeoutException ste) {
+ // nobody connected to us. Go ahead and call closeQuietly just to make sure we don't leave
+ // any sockets lingering
+ SocketUtils.closeQuietly(socket);
+ continue;
+ } catch (final SocketException se) {
+ logger.warn("Failed to communicate with " + (socket == null ? "Unknown Host" : socket.getInetAddress().getHostName()) + " due to " + se, se);
+ SocketUtils.closeQuietly(socket);
+ continue;
+ } catch (final Throwable t) {
+ logger.warn("Socket Listener encountered exception: " + t, t);
+ SocketUtils.closeQuietly(socket);
+ continue;
+ }
+
+ final Socket finalSocket = socket;
+ runnableExecServiceRef.execute(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ dispatchRequest(finalSocket);
+ } catch (final Throwable t) {
+ logger.warn("Dispatching socket request encountered exception due to: " + t, t);
+ } finally {
+ SocketUtils.closeQuietly(finalSocket);
+ }
+ }
+ });
+ } catch (final Throwable t) {
+ logger.error("Socket Listener encountered exception: " + t, t);
+ SocketUtils.closeQuietly(socket);
+ }
+ }
+ }
+ });
+ t.setName("Cluster Socket Listener");
+ t.start();
+ }
+
+ public boolean isRunning() {
+ return (executorService != null && executorService.isShutdown() == false);
+ }
+
+ public void stop() throws IOException {
+
+ if (isRunning() == false) {
+ return;
+ }
+
+ // shutdown executor service
+ try {
+ if (getShutdownListenerSeconds() <= 0) {
+ executorService.shutdownNow();
+ } else {
+ executorService.shutdown();
+ }
+ executorService.awaitTermination(getShutdownListenerSeconds(), TimeUnit.SECONDS);
+ } catch (final InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ } finally {
+ if (executorService.isTerminated()) {
+ logger.info("Socket Listener has been terminated successfully.");
+ } else {
+ logger.warn("Socket Listener has not terminated properly. There exists an uninterruptable thread that will take an indeterminate amount of time to stop.");
+ }
+ }
+
+ // shutdown server socket
+ SocketUtils.closeQuietly(serverSocket);
+
+ }
+
+ public int getShutdownListenerSeconds() {
+ return shutdownListenerSeconds.get();
+ }
+
+ public void setShutdownListenerSeconds(final int shutdownListenerSeconds) {
+ this.shutdownListenerSeconds.set(shutdownListenerSeconds);
+ }
+
+ public ServerSocketConfiguration getConfiguration() {
+ return configuration;
+ }
+
+ public int getPort() {
+ if (isRunning()) {
+ return serverSocket.getLocalPort();
+ } else {
+ return port;
+ }
+ }
+
+}
[33/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-flowfile-packager/src/main/java/org/apache/nifi/util/FlowFileUnpackagerV3.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-flowfile-packager/src/main/java/org/apache/nifi/util/FlowFileUnpackagerV3.java b/nifi-commons/nifi-flowfile-packager/src/main/java/org/apache/nifi/util/FlowFileUnpackagerV3.java
new file mode 100644
index 0000000..f937585
--- /dev/null
+++ b/nifi-commons/nifi-flowfile-packager/src/main/java/org/apache/nifi/util/FlowFileUnpackagerV3.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+public class FlowFileUnpackagerV3 implements FlowFileUnpackager {
+
+ private byte[] nextHeader = null;
+ private boolean haveReadSomething = false;
+ private final byte readBuffer[] = new byte[8192];
+
+ @Override
+ public boolean hasMoreData() throws IOException {
+ return nextHeader != null || !haveReadSomething;
+ }
+
+ private byte[] readHeader(final InputStream in) throws IOException {
+ final byte[] header = new byte[FlowFilePackagerV3.MAGIC_HEADER.length];
+ for (int i = 0; i < header.length; i++) {
+ final int next = in.read();
+ if (next < 0) {
+ if (i == 0) {
+ return null;
+ }
+
+ throw new IOException("Not in FlowFile-v3 format");
+ }
+ header[i] = (byte) (next & 0xFF);
+ }
+
+ return header;
+ }
+
+ @Override
+ public Map<String, String> unpackageFlowFile(final InputStream in, final OutputStream out) throws IOException {
+ final byte[] header = (nextHeader == null) ? readHeader(in) : nextHeader;
+ if (!Arrays.equals(header, FlowFilePackagerV3.MAGIC_HEADER)) {
+ throw new IOException("Not in FlowFile-v3 format");
+ }
+
+ final Map<String, String> attributes = readAttributes(in);
+ final long expectedNumBytes = readLong(in); // read length of payload
+ copy(in, out, expectedNumBytes); // read payload
+
+ nextHeader = readHeader(in);
+ haveReadSomething = true;
+
+ return attributes;
+ }
+
+ protected Map<String, String> readAttributes(final InputStream in) throws IOException {
+ final Map<String, String> attributes = new HashMap<>();
+ final Integer numAttributes = readFieldLength(in); //read number of attributes
+ if (numAttributes == null) {
+ return null;
+ }
+ if (numAttributes == 0) {
+ throw new IOException("flow files cannot have zero attributes");
+ }
+ for (int i = 0; i < numAttributes; i++) { //read each attribute key/value pair
+ final String key = readString(in);
+ final String value = readString(in);
+ attributes.put(key, value);
+ }
+
+ return attributes;
+ }
+
+ protected String readString(final InputStream in) throws IOException {
+ final Integer numBytes = readFieldLength(in);
+ if (numBytes == null) {
+ throw new EOFException();
+ }
+ final byte[] bytes = new byte[numBytes];
+ fillBuffer(in, bytes, numBytes);
+ return new String(bytes, "UTF-8");
+ }
+
+ private void fillBuffer(final InputStream in, final byte[] buffer, final int length) throws IOException {
+ int bytesRead;
+ int totalBytesRead = 0;
+ while ((bytesRead = in.read(buffer, totalBytesRead, length - totalBytesRead)) > 0) {
+ totalBytesRead += bytesRead;
+ }
+ if (totalBytesRead != length) {
+ throw new EOFException();
+ }
+ }
+
+ protected long copy(final InputStream in, final OutputStream out, final long numBytes) throws IOException {
+ int bytesRead;
+ long totalBytesRead = 0L;
+ while ((bytesRead = in.read(readBuffer, 0, (int) Math.min(readBuffer.length, numBytes - totalBytesRead))) > 0) {
+ out.write(readBuffer, 0, bytesRead);
+ totalBytesRead += bytesRead;
+ }
+
+ if (totalBytesRead < numBytes) {
+ throw new EOFException("Expected " + numBytes + " but received " + totalBytesRead);
+ }
+
+ return totalBytesRead;
+ }
+
+ protected long readLong(final InputStream in) throws IOException {
+ fillBuffer(in, readBuffer, 8);
+ return (((long) readBuffer[0] << 56)
+ + ((long) (readBuffer[1] & 255) << 48)
+ + ((long) (readBuffer[2] & 255) << 40)
+ + ((long) (readBuffer[3] & 255) << 32)
+ + ((long) (readBuffer[4] & 255) << 24)
+ + ((readBuffer[5] & 255) << 16)
+ + ((readBuffer[6] & 255) << 8)
+ + ((readBuffer[7] & 255)));
+ }
+
+ private Integer readFieldLength(final InputStream in) throws IOException {
+ final int firstValue = in.read();
+ final int secondValue = in.read();
+ if (firstValue < 0) {
+ return null;
+ }
+ if (secondValue < 0) {
+ throw new EOFException();
+ }
+ if (firstValue == 0xff && secondValue == 0xff) {
+ int ch1 = in.read();
+ int ch2 = in.read();
+ int ch3 = in.read();
+ int ch4 = in.read();
+ if ((ch1 | ch2 | ch3 | ch4) < 0) {
+ throw new EOFException();
+ }
+ return ((ch1 << 24) + (ch2 << 16) + (ch3 << 8) + (ch4));
+ } else {
+ return ((firstValue << 8) + (secondValue));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-flowfile-packager/src/main/java/org/apache/nifi/util/Unpackage.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-flowfile-packager/src/main/java/org/apache/nifi/util/Unpackage.java b/nifi-commons/nifi-flowfile-packager/src/main/java/org/apache/nifi/util/Unpackage.java
new file mode 100644
index 0000000..19f702c
--- /dev/null
+++ b/nifi-commons/nifi-flowfile-packager/src/main/java/org/apache/nifi/util/Unpackage.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.Map;
+import java.util.UUID;
+
+public class Unpackage {
+
+ private static void printUsage() {
+ System.out.println("java " + Unpackage.class.getCanonicalName() + " <version> <input file 1> [<input file 2> <input file 3> ... <input file N>]");
+ System.out.println("<version> : The version of the FlowFile Package format. Valid values are 1, 2, 3");
+ System.out.println("<input file X> : The FlowFile package to unpack");
+ System.out.println();
+ }
+
+ public static void main(final String[] args) throws IOException {
+ if (args.length < 2) {
+ printUsage();
+ return;
+ }
+
+ final String version = args[0];
+
+ int inputFileCount = 0;
+ int outputFileCount = 0;
+
+ for (int i = 1; i < args.length; i++) {
+ final String filename = args[i];
+ final File inFile = new File(filename);
+
+ if (inFile.isDirectory()) {
+ System.out.println("WARNING: input file " + inFile + " is a directory; skipping");
+ continue;
+ }
+
+ if (!inFile.exists() || !inFile.canRead()) {
+ System.out.println("ERROR: unable to read file " + inFile);
+ continue;
+ }
+
+ final File outputDir = new File(inFile.getAbsolutePath() + ".unpacked");
+ if (!outputDir.exists() && !outputDir.mkdir()) {
+ System.out.println("ERROR: Unable to create directory " + outputDir);
+ continue;
+ }
+
+ final File tempFile = new File(outputDir, ".temp." + UUID.randomUUID().toString() + ".unpackage");
+ inputFileCount++;
+ try (final FileInputStream fis = new FileInputStream(inFile);
+ final BufferedInputStream bufferedIn = new BufferedInputStream(fis)) {
+
+ final FlowFileUnpackager unpackager = createUnpackager(version);
+ while (unpackager.hasMoreData()) {
+ outputFileCount++;
+ final Map<String, String> attributes;
+
+ try (final FileOutputStream fos = new FileOutputStream(tempFile);
+ final BufferedOutputStream bufferedOut = new BufferedOutputStream(fos)) {
+ attributes = unpackager.unpackageFlowFile(bufferedIn, bufferedOut);
+ }
+
+ String outputFilename = attributes.get("filename");
+ if (outputFilename == null) {
+ outputFilename = attributes.get("nf.file.name");
+ }
+
+ final File outputFile = new File(outputDir, outputFilename);
+ tempFile.renameTo(outputFile);
+
+ final File attributeFilename = new File(outputDir, outputFilename + ".attributes");
+ try (final FileOutputStream fos = new FileOutputStream(attributeFilename);
+ final BufferedOutputStream bufferedOut = new BufferedOutputStream(fos)) {
+
+ for (final Map.Entry<String, String> entry : attributes.entrySet()) {
+ bufferedOut.write((entry.getKey() + "=" + entry.getValue() + "\n").getBytes("UTF-8"));
+ }
+ }
+ }
+ }
+ }
+
+ System.out.println("Unpacked " + inputFileCount + " packages into " + outputFileCount + " files");
+ }
+
+ public static FlowFileUnpackager createUnpackager(final String version) {
+ switch (version) {
+ case "1":
+ return new FlowFileUnpackagerV1();
+ case "2":
+ return new FlowFileUnpackagerV2();
+ case "3":
+ return new FlowFileUnpackagerV3();
+ default:
+ System.out.println("ERROR: Invalid version: " + version + "; must be 1, 2, or 3");
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-flowfile-packager/src/test/java/org/apache/nifi/util/TestPackageUnpackageV3.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-flowfile-packager/src/test/java/org/apache/nifi/util/TestPackageUnpackageV3.java b/nifi-commons/nifi-flowfile-packager/src/test/java/org/apache/nifi/util/TestPackageUnpackageV3.java
new file mode 100644
index 0000000..24cd374
--- /dev/null
+++ b/nifi-commons/nifi-flowfile-packager/src/test/java/org/apache/nifi/util/TestPackageUnpackageV3.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.junit.Test;
+
+public class TestPackageUnpackageV3 {
+
+ @Test
+ public void test() throws IOException {
+ final FlowFilePackager packager = new FlowFilePackagerV3();
+ final FlowFileUnpackager unpackager = new FlowFileUnpackagerV3();
+
+ final byte[] data = "Hello, World!".getBytes("UTF-8");
+ final Map<String, String> map = new HashMap<>();
+ map.put("abc", "cba");
+
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ final ByteArrayInputStream in = new ByteArrayInputStream(data);
+ packager.packageFlowFile(in, baos, map, data.length);
+
+ final byte[] encoded = baos.toByteArray();
+ final ByteArrayInputStream encodedIn = new ByteArrayInputStream(encoded);
+ final ByteArrayOutputStream decodedOut = new ByteArrayOutputStream();
+ final Map<String, String> unpackagedAttributes = unpackager.unpackageFlowFile(encodedIn, decodedOut);
+ final byte[] decoded = decodedOut.toByteArray();
+
+ assertEquals(map, unpackagedAttributes);
+ assertTrue(Arrays.equals(data, decoded));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/.gitignore
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/.gitignore b/nifi-commons/nifi-hl7-query-language/.gitignore
new file mode 100644
index 0000000..e91d5c4
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/.gitignore
@@ -0,0 +1,3 @@
+/target/
+/target/
+/target/
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/pom.xml b/nifi-commons/nifi-hl7-query-language/pom.xml
new file mode 100644
index 0000000..8b04e5c
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/pom.xml
@@ -0,0 +1,127 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-commons</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-hl7-query-language</artifactId>
+ <packaging>jar</packaging>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>1.7</source>
+ <target>1.7</target>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.antlr</groupId>
+ <artifactId>antlr3-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>antlr</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes combine.children="append">
+ <exclude>src/test/resources/hypoglycemia</exclude>
+ <exclude>src/test/resources/hyperglycemia</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <configuration>
+ <excludes>**/HL7QueryParser.java,**/HL7QueryLexer.java</excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.antlr</groupId>
+ <artifactId>antlr-runtime</artifactId>
+ </dependency>
+
+ <!-- HAPI to parse v2 messages -->
+ <dependency>
+ <groupId>ca.uhn.hapi</groupId>
+ <artifactId>hapi-base</artifactId>
+ <version>2.2</version>
+ </dependency>
+ <dependency>
+ <groupId>ca.uhn.hapi</groupId>
+ <artifactId>hapi-structures-v21</artifactId>
+ <version>2.2</version>
+ </dependency>
+ <dependency>
+ <groupId>ca.uhn.hapi</groupId>
+ <artifactId>hapi-structures-v22</artifactId>
+ <version>2.2</version>
+ </dependency>
+ <dependency>
+ <groupId>ca.uhn.hapi</groupId>
+ <artifactId>hapi-structures-v23</artifactId>
+ <version>2.2</version>
+ </dependency>
+ <dependency>
+ <groupId>ca.uhn.hapi</groupId>
+ <artifactId>hapi-structures-v231</artifactId>
+ <version>2.2</version>
+ </dependency>
+ <dependency>
+ <groupId>ca.uhn.hapi</groupId>
+ <artifactId>hapi-structures-v24</artifactId>
+ <version>2.2</version>
+ </dependency>
+ <dependency>
+ <groupId>ca.uhn.hapi</groupId>
+ <artifactId>hapi-structures-v25</artifactId>
+ <version>2.2</version>
+ </dependency>
+ <dependency>
+ <groupId>ca.uhn.hapi</groupId>
+ <artifactId>hapi-structures-v251</artifactId>
+ <version>2.2</version>
+ </dependency>
+ <dependency>
+ <groupId>ca.uhn.hapi</groupId>
+ <artifactId>hapi-structures-v26</artifactId>
+ <version>2.2</version>
+ </dependency>
+
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/antlr3/org/apache/nifi/hl7/query/antlr/HL7QueryLexer.g
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/antlr3/org/apache/nifi/hl7/query/antlr/HL7QueryLexer.g b/nifi-commons/nifi-hl7-query-language/src/main/antlr3/org/apache/nifi/hl7/query/antlr/HL7QueryLexer.g
new file mode 100644
index 0000000..478028b
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/antlr3/org/apache/nifi/hl7/query/antlr/HL7QueryLexer.g
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+lexer grammar HL7QueryLexer;
+
+@header {
+ package org.apache.nifi.hl7.query.antlr;
+ import org.apache.nifi.hl7.query.exception.HL7QueryParsingException;
+}
+
+@rulecatch {
+ catch(final Exception e) {
+ throw new HL7QueryParsingException(e);
+ }
+}
+
+@members {
+ public void displayRecognitionError(String[] tokenNames, RecognitionException e) {
+ final StringBuilder sb = new StringBuilder();
+ if ( e.token == null ) {
+ sb.append("Unrecognized token ");
+ } else {
+ sb.append("Unexpected token '").append(e.token.getText()).append("' ");
+ }
+ sb.append("at line ").append(e.line);
+ if ( e.approximateLineInfo ) {
+ sb.append(" (approximately)");
+ }
+ sb.append(", column ").append(e.charPositionInLine);
+ sb.append(". Query: ").append(e.input.toString());
+
+ throw new HL7QueryParsingException(sb.toString());
+ }
+
+ public void recover(RecognitionException e) {
+ final StringBuilder sb = new StringBuilder();
+ if ( e.token == null ) {
+ sb.append("Unrecognized token ");
+ } else {
+ sb.append("Unexpected token '").append(e.token.getText()).append("' ");
+ }
+ sb.append("at line ").append(e.line);
+ if ( e.approximateLineInfo ) {
+ sb.append(" (approximately)");
+ }
+ sb.append(", column ").append(e.charPositionInLine);
+ sb.append(". Query: ").append(e.input.toString());
+
+ throw new HL7QueryParsingException(sb.toString());
+ }
+}
+
+
+// PUNCTUATION & SPECIAL CHARACTERS
+WHITESPACE : (' '|'\t'|'\n'|'\r')+ { $channel = HIDDEN; };
+COMMENT : '#' ( ~('\n') )* '\n' { $channel = HIDDEN; };
+
+LPAREN : '(';
+RPAREN : ')';
+LBRACE : '{';
+RBRACE : '}';
+COLON : ':';
+COMMA : ',';
+DOT : '.';
+SEMICOLON : ';';
+
+
+
+// OPERATORS
+EQUALS : '=';
+NOT_EQUALS : '!=';
+GT : '>';
+GE : '>=';
+LT : '<';
+LE : '<=';
+REGEX : 'MATCHES REGEX';
+LIKE : 'LIKE';
+IS_NULL : 'IS NULL';
+NOT_NULL : 'NOT NULL';
+
+
+// KEYWORDS
+AND : 'AND';
+OR : 'OR';
+NOT : 'NOT';
+
+TRUE : 'true';
+FALSE : 'false';
+
+SELECT : 'select' | 'SELECT';
+DECLARE : 'declare' | 'DECLARE';
+OPTIONAL : 'optional' | 'OPTIONAL';
+REQUIRED : 'required' | 'REQUIRED';
+AS : 'as' | 'AS';
+WHERE : 'where' | 'WHERE';
+
+MESSAGE : 'MESSAGE' | 'message';
+SEGMENT : 'SEGMENT' | 'segment';
+
+
+SEGMENT_NAME : LETTER ALPHA_NUMERIC ALPHA_NUMERIC;
+
+
+NUMBER : ('0'..'9')+;
+fragment LETTER : 'A'..'Z';
+fragment ALPHA_NUMERIC : 'A'..'Z' | '0'..'9';
+
+
+// STRINGS
+STRING_LITERAL
+@init{StringBuilder lBuf = new StringBuilder();}
+ :
+ (
+ '"'
+ (
+ escaped=ESC {lBuf.append(getText());} |
+ normal = ~( '"' | '\\' | '\n' | '\r' | '\t' ) { lBuf.appendCodePoint(normal);}
+ )*
+ '"'
+ )
+ {
+ setText(lBuf.toString());
+ }
+ |
+ (
+ '\''
+ (
+ escaped=ESC {lBuf.append(getText());} |
+ normal = ~( '\'' | '\\' | '\n' | '\r' | '\t' ) { lBuf.appendCodePoint(normal);}
+ )*
+ '\''
+ )
+ {
+ setText(lBuf.toString());
+ }
+ ;
+
+
+fragment
+ESC
+ : '\\'
+ (
+ '"' { setText("\""); }
+ | '\'' { setText("\'"); }
+ | 'r' { setText("\r"); }
+ | 'n' { setText("\n"); }
+ | 't' { setText("\t"); }
+ | '\\' { setText("\\\\"); }
+ | nextChar = ~('"' | '\'' | 'r' | 'n' | 't' | '\\')
+ {
+ StringBuilder lBuf = new StringBuilder(); lBuf.append("\\\\").appendCodePoint(nextChar); setText(lBuf.toString());
+ }
+ )
+ ;
+
+IDENTIFIER : (
+ ~('$' | '{' | '}' | '(' | ')' | '[' | ']' | ',' | ':' | ';' | '/' | '*' | '\'' | ' ' | '\t' | '\r' | '\n' | '0'..'9' | '.')
+ ~('$' | '{' | '}' | '(' | ')' | '[' | ']' | ',' | ':' | ';' | '/' | '*' | '\'' | ' ' | '\t' | '\r' | '\n' | '.')*
+ );
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/antlr3/org/apache/nifi/hl7/query/antlr/HL7QueryParser.g
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/antlr3/org/apache/nifi/hl7/query/antlr/HL7QueryParser.g b/nifi-commons/nifi-hl7-query-language/src/main/antlr3/org/apache/nifi/hl7/query/antlr/HL7QueryParser.g
new file mode 100644
index 0000000..f051872
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/antlr3/org/apache/nifi/hl7/query/antlr/HL7QueryParser.g
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parser grammar HL7QueryParser;
+
+options {
+ output=AST;
+ tokenVocab=HL7QueryLexer;
+}
+
+tokens {
+ QUERY;
+ DECLARATION;
+}
+
+@header {
+ package org.apache.nifi.hl7.query.antlr;
+ import org.apache.nifi.hl7.query.exception.HL7QueryParsingException;
+}
+
+@members {
+ public void displayRecognitionError(String[] tokenNames, RecognitionException e) {
+ final StringBuilder sb = new StringBuilder();
+ if ( e.token == null ) {
+ sb.append("Unrecognized token ");
+ } else {
+ sb.append("Unexpected token '").append(e.token.getText()).append("' ");
+ }
+ sb.append("at line ").append(e.line);
+ if ( e.approximateLineInfo ) {
+ sb.append(" (approximately)");
+ }
+ sb.append(", column ").append(e.charPositionInLine);
+ sb.append(". Query: ").append(e.input.toString());
+
+ throw new HL7QueryParsingException(sb.toString());
+ }
+
+ public void recover(final RecognitionException e) {
+ final StringBuilder sb = new StringBuilder();
+ if ( e.token == null ) {
+ sb.append("Unrecognized token ");
+ } else {
+ sb.append("Unexpected token '").append(e.token.getText()).append("' ");
+ }
+ sb.append("at line ").append(e.line);
+ if ( e.approximateLineInfo ) {
+ sb.append(" (approximately)");
+ }
+ sb.append(", column ").append(e.charPositionInLine);
+ sb.append(". Query: ").append(e.input.toString());
+
+ throw new HL7QueryParsingException(sb.toString());
+ }
+}
+
+
+declareClause : DECLARE^ declaration (COMMA! declaration)*;
+
+requiredOrOptional : REQUIRED | OPTIONAL;
+declaration : IDENTIFIER AS requiredOrOptional SEGMENT_NAME ->
+ ^(DECLARATION IDENTIFIER requiredOrOptional SEGMENT_NAME);
+
+
+selectClause : SELECT^ selectableClause;
+selectableClause : selectable (COMMA! selectable)*;
+selectable : (MESSAGE | ref | field)^ (AS! IDENTIFIER^)?;
+
+
+whereClause : WHERE^ conditions;
+
+conditions : condition ((AND^ | OR^) condition)*;
+
+condition : NOT^ condition | LPAREN! conditions RPAREN! | evaluation;
+
+evaluation : expression
+ (
+ unaryOperator^
+ | (binaryOperator^ expression)
+ );
+
+expression : (LPAREN! expr RPAREN!) | expr;
+expr : ref | field | STRING_LITERAL | NUMBER;
+
+unaryOperator : IS_NULL | NOT_NULL;
+binaryOperator : EQUALS | NOT_EQUALS | LT | GT | LE | GE;
+
+ref : (SEGMENT_NAME | IDENTIFIER);
+field : ref DOT^ NUMBER
+ (DOT^ NUMBER (DOT^ NUMBER (DOT^ NUMBER)?)?)?;
+
+
+query : declareClause? selectClause whereClause? EOF ->
+ ^(QUERY declareClause? selectClause whereClause?);
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/hapi/EmptyField.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/hapi/EmptyField.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/hapi/EmptyField.java
new file mode 100644
index 0000000..dc73efb
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/hapi/EmptyField.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.hapi;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.nifi.hl7.model.HL7Component;
+import org.apache.nifi.hl7.model.HL7Field;
+
+public class EmptyField implements HL7Field {
+
+ @Override
+ public String getValue() {
+ return null;
+ }
+
+ @Override
+ public List<HL7Component> getComponents() {
+ return Collections.emptyList();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/hapi/HapiField.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/hapi/HapiField.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/hapi/HapiField.java
new file mode 100644
index 0000000..94cce5c
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/hapi/HapiField.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.hapi;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.nifi.hl7.model.HL7Component;
+import org.apache.nifi.hl7.model.HL7Field;
+
+import ca.uhn.hl7v2.model.Composite;
+import ca.uhn.hl7v2.model.ExtraComponents;
+import ca.uhn.hl7v2.model.Primitive;
+import ca.uhn.hl7v2.model.Type;
+import ca.uhn.hl7v2.model.Varies;
+import ca.uhn.hl7v2.parser.EncodingCharacters;
+import ca.uhn.hl7v2.parser.PipeParser;
+
+public class HapiField implements HL7Field, HL7Component {
+
+ private final String value;
+ private final List<HL7Component> components;
+
+ public HapiField(final Type type) {
+ this.value = PipeParser.encode(type, EncodingCharacters.defaultInstance());
+
+ final List<HL7Component> componentList = new ArrayList<>();
+ if (type instanceof Composite) {
+ final Composite composite = (Composite) type;
+
+ for (final Type component : composite.getComponents()) {
+ componentList.add(new HapiField(component));
+ }
+ }
+
+ final ExtraComponents extra = type.getExtraComponents();
+ if (extra != null && extra.numComponents() > 0) {
+ final String singleFieldValue;
+ if (type instanceof Primitive) {
+ singleFieldValue = ((Primitive) type).getValue();
+ } else {
+ singleFieldValue = this.value;
+ }
+ componentList.add(new SingleValueField(singleFieldValue));
+
+ for (int i = 0; i < extra.numComponents(); i++) {
+ final Varies varies = extra.getComponent(i);
+ componentList.add(new HapiField(varies));
+ }
+ }
+
+ this.components = Collections.unmodifiableList(componentList);
+ }
+
+ @Override
+ public String getValue() {
+ return value;
+ }
+
+ @Override
+ public List<HL7Component> getComponents() {
+ return components;
+ }
+
+ @Override
+ public String toString() {
+ return value;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/hapi/HapiMessage.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/hapi/HapiMessage.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/hapi/HapiMessage.java
new file mode 100644
index 0000000..74a8f6c
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/hapi/HapiMessage.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.hapi;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.nifi.hl7.model.HL7Message;
+import org.apache.nifi.hl7.model.HL7Segment;
+
+import ca.uhn.hl7v2.HL7Exception;
+import ca.uhn.hl7v2.model.Group;
+import ca.uhn.hl7v2.model.Message;
+import ca.uhn.hl7v2.model.Segment;
+import ca.uhn.hl7v2.model.Structure;
+
+public class HapiMessage implements HL7Message {
+
+ private final Message message;
+ private final List<HL7Segment> allSegments;
+ private final Map<String, List<HL7Segment>> segmentMap;
+
+ public HapiMessage(final Message message) throws HL7Exception {
+ this.message = message;
+
+ allSegments = new ArrayList<>();
+ populateSegments(message, allSegments);
+
+ segmentMap = new HashMap<>();
+ for (final HL7Segment segment : allSegments) {
+ final String segmentName = segment.getName();
+ List<HL7Segment> segmentList = segmentMap.get(segmentName);
+ if (segmentList == null) {
+ segmentList = new ArrayList<>();
+ segmentMap.put(segmentName, segmentList);
+ }
+
+ segmentList.add(segment);
+ }
+ }
+
+ private void populateSegments(final Group group, final List<HL7Segment> segments) throws HL7Exception {
+ for (final String structureName : group.getNames()) {
+ final Structure[] structures = group.getAll(structureName);
+ if (group.isGroup(structureName)) {
+ for (final Structure structure : structures) {
+ populateSegments((Group) structure, segments);
+ }
+ } else {
+ for (final Structure structure : structures) {
+ final Segment segment = (Segment) structure;
+ final HapiSegment hapiSegment = new HapiSegment(segment);
+ segments.add(hapiSegment);
+ }
+ }
+ }
+ }
+
+ @Override
+ public List<HL7Segment> getSegments() {
+ return Collections.unmodifiableList(allSegments);
+ }
+
+ @Override
+ public List<HL7Segment> getSegments(final String segmentType) {
+ final List<HL7Segment> segments = segmentMap.get(segmentType);
+ if (segments == null) {
+ return Collections.emptyList();
+ }
+
+ return Collections.unmodifiableList(segments);
+ }
+
+ @Override
+ public String toString() {
+ return message.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/hapi/HapiSegment.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/hapi/HapiSegment.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/hapi/HapiSegment.java
new file mode 100644
index 0000000..20e646c
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/hapi/HapiSegment.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.hapi;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.nifi.hl7.model.HL7Field;
+import org.apache.nifi.hl7.model.HL7Segment;
+
+import ca.uhn.hl7v2.HL7Exception;
+import ca.uhn.hl7v2.model.Segment;
+import ca.uhn.hl7v2.model.Type;
+
+public class HapiSegment implements HL7Segment {
+
+ private final Segment segment;
+ private final List<HL7Field> fields;
+
+ public HapiSegment(final Segment segment) throws HL7Exception {
+ this.segment = segment;
+
+ final List<HL7Field> fieldList = new ArrayList<>();
+ for (int i = 1; i <= segment.numFields(); i++) {
+ final Type[] types = segment.getField(i);
+
+ if (types == null || types.length == 0) {
+ fieldList.add(new EmptyField());
+ continue;
+ }
+
+ for (final Type type : types) {
+ fieldList.add(new HapiField(type));
+ }
+ }
+
+ this.fields = Collections.unmodifiableList(fieldList);
+ }
+
+ @Override
+ public String getName() {
+ return segment.getName();
+ }
+
+ @Override
+ public List<HL7Field> getFields() {
+ return fields;
+ }
+
+ @Override
+ public String toString() {
+ return segment.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/hapi/SingleValueField.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/hapi/SingleValueField.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/hapi/SingleValueField.java
new file mode 100644
index 0000000..b8a6351
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/hapi/SingleValueField.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.hapi;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.nifi.hl7.model.HL7Component;
+import org.apache.nifi.hl7.model.HL7Field;
+
+public class SingleValueField implements HL7Field {
+
+ private final String value;
+
+ public SingleValueField(final String value) {
+ this.value = value;
+ }
+
+ @Override
+ public String getValue() {
+ return value;
+ }
+
+ @Override
+ public List<HL7Component> getComponents() {
+ return Collections.emptyList();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/io/HL7Reader.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/io/HL7Reader.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/io/HL7Reader.java
new file mode 100644
index 0000000..4d2f276
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/io/HL7Reader.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.io;
+
+import java.io.IOException;
+
+import org.apache.nifi.hl7.model.HL7Message;
+
+public interface HL7Reader {
+
+ HL7Message nextMessage() throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/io/exception/InvalidHL7Exception.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/io/exception/InvalidHL7Exception.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/io/exception/InvalidHL7Exception.java
new file mode 100644
index 0000000..c0c74f9
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/io/exception/InvalidHL7Exception.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.io.exception;
+
+import java.io.IOException;
+
+public class InvalidHL7Exception extends IOException {
+
+ private static final long serialVersionUID = -5675416667224562441L;
+
+ public InvalidHL7Exception() {
+ super();
+ }
+
+ public InvalidHL7Exception(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public InvalidHL7Exception(String message) {
+ super(message);
+ }
+
+ public InvalidHL7Exception(Throwable cause) {
+ super(cause);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/model/HL7Component.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/model/HL7Component.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/model/HL7Component.java
new file mode 100644
index 0000000..d8c1495
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/model/HL7Component.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.model;
+
+import java.util.List;
+
+public interface HL7Component {
+
+ String getValue();
+
+ List<HL7Component> getComponents();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/model/HL7Field.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/model/HL7Field.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/model/HL7Field.java
new file mode 100644
index 0000000..ff58d96
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/model/HL7Field.java
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.model;
+
+public interface HL7Field extends HL7Component {
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/model/HL7Message.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/model/HL7Message.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/model/HL7Message.java
new file mode 100644
index 0000000..dc68c2b
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/model/HL7Message.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.model;
+
+import java.util.List;
+
+public interface HL7Message {
+
+ List<HL7Segment> getSegments();
+
+ List<HL7Segment> getSegments(String segmentType);
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/model/HL7Segment.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/model/HL7Segment.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/model/HL7Segment.java
new file mode 100644
index 0000000..d5d67ae
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/model/HL7Segment.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.model;
+
+import java.util.List;
+
+public interface HL7Segment {
+
+ String getName();
+
+ List<HL7Field> getFields();
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/Declaration.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/Declaration.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/Declaration.java
new file mode 100644
index 0000000..94998e7
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/Declaration.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.query;
+
+import org.apache.nifi.hl7.model.HL7Message;
+
+public interface Declaration {
+
+ String getAlias();
+
+ boolean isRequired();
+
+ Object getDeclaredValue(HL7Message message);
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/HL7Query.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/HL7Query.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/HL7Query.java
new file mode 100644
index 0000000..3734b4f
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/HL7Query.java
@@ -0,0 +1,384 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.query;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.antlr.runtime.ANTLRStringStream;
+import org.antlr.runtime.CharStream;
+import org.antlr.runtime.CommonTokenStream;
+import org.antlr.runtime.tree.Tree;
+import org.apache.nifi.hl7.model.HL7Message;
+import org.apache.nifi.hl7.query.evaluator.BooleanEvaluator;
+import org.apache.nifi.hl7.query.evaluator.Evaluator;
+import org.apache.nifi.hl7.query.evaluator.IntegerEvaluator;
+import org.apache.nifi.hl7.query.evaluator.comparison.EqualsEvaluator;
+import org.apache.nifi.hl7.query.evaluator.comparison.GreaterThanEvaluator;
+import org.apache.nifi.hl7.query.evaluator.comparison.GreaterThanOrEqualEvaluator;
+import org.apache.nifi.hl7.query.evaluator.comparison.IsNullEvaluator;
+import org.apache.nifi.hl7.query.evaluator.comparison.LessThanEvaluator;
+import org.apache.nifi.hl7.query.evaluator.comparison.LessThanOrEqualEvaluator;
+import org.apache.nifi.hl7.query.evaluator.comparison.NotEqualsEvaluator;
+import org.apache.nifi.hl7.query.evaluator.comparison.NotEvaluator;
+import org.apache.nifi.hl7.query.evaluator.comparison.NotNullEvaluator;
+import org.apache.nifi.hl7.query.evaluator.literal.IntegerLiteralEvaluator;
+import org.apache.nifi.hl7.query.evaluator.literal.StringLiteralEvaluator;
+import org.apache.nifi.hl7.query.evaluator.logic.AndEvaluator;
+import org.apache.nifi.hl7.query.evaluator.logic.OrEvaluator;
+import org.apache.nifi.hl7.query.evaluator.message.DeclaredReferenceEvaluator;
+import org.apache.nifi.hl7.query.evaluator.message.DotEvaluator;
+import org.apache.nifi.hl7.query.evaluator.message.MessageEvaluator;
+import org.apache.nifi.hl7.query.evaluator.message.SegmentEvaluator;
+import org.apache.nifi.hl7.query.exception.HL7QueryParsingException;
+import org.apache.nifi.hl7.query.result.MissedResult;
+import org.apache.nifi.hl7.query.result.StandardQueryResult;
+
+import org.apache.nifi.hl7.query.antlr.HL7QueryLexer;
+import org.apache.nifi.hl7.query.antlr.HL7QueryParser;
+import static org.apache.nifi.hl7.query.antlr.HL7QueryParser.AND;
+import static org.apache.nifi.hl7.query.antlr.HL7QueryParser.DECLARE;
+import static org.apache.nifi.hl7.query.antlr.HL7QueryParser.DOT;
+import static org.apache.nifi.hl7.query.antlr.HL7QueryParser.EQUALS;
+import static org.apache.nifi.hl7.query.antlr.HL7QueryParser.GE;
+import static org.apache.nifi.hl7.query.antlr.HL7QueryParser.GT;
+import static org.apache.nifi.hl7.query.antlr.HL7QueryParser.IDENTIFIER;
+import static org.apache.nifi.hl7.query.antlr.HL7QueryParser.IS_NULL;
+import static org.apache.nifi.hl7.query.antlr.HL7QueryParser.LE;
+import static org.apache.nifi.hl7.query.antlr.HL7QueryParser.LT;
+import static org.apache.nifi.hl7.query.antlr.HL7QueryParser.MESSAGE;
+import static org.apache.nifi.hl7.query.antlr.HL7QueryParser.NOT;
+import static org.apache.nifi.hl7.query.antlr.HL7QueryParser.NOT_EQUALS;
+import static org.apache.nifi.hl7.query.antlr.HL7QueryParser.NOT_NULL;
+import static org.apache.nifi.hl7.query.antlr.HL7QueryParser.NUMBER;
+import static org.apache.nifi.hl7.query.antlr.HL7QueryParser.OR;
+import static org.apache.nifi.hl7.query.antlr.HL7QueryParser.REQUIRED;
+import static org.apache.nifi.hl7.query.antlr.HL7QueryParser.SEGMENT_NAME;
+import static org.apache.nifi.hl7.query.antlr.HL7QueryParser.SELECT;
+import static org.apache.nifi.hl7.query.antlr.HL7QueryParser.STRING_LITERAL;
+import static org.apache.nifi.hl7.query.antlr.HL7QueryParser.WHERE;
+
+public class HL7Query {
+
+ private final Tree tree;
+ private final String query;
+ private final Set<Declaration> declarations = new HashSet<>();
+
+ private final List<Selection> selections;
+ private final BooleanEvaluator whereEvaluator;
+
+ private HL7Query(final Tree tree, final String query) {
+ this.tree = tree;
+ this.query = query;
+
+ List<Selection> select = null;
+ BooleanEvaluator where = null;
+ for (int i = 0; i < tree.getChildCount(); i++) {
+ final Tree child = tree.getChild(i);
+
+ switch (child.getType()) {
+ case DECLARE:
+ processDeclare(child);
+ break;
+ case SELECT:
+ select = processSelect(child);
+ break;
+ case WHERE:
+ where = processWhere(child);
+ break;
+ default:
+ throw new HL7QueryParsingException("Found unexpected clause at root level: " + tree.getText());
+ }
+ }
+
+ this.whereEvaluator = where;
+ this.selections = select;
+ }
+
+ private void processDeclare(final Tree declare) {
+ for (int i = 0; i < declare.getChildCount(); i++) {
+ final Tree declarationTree = declare.getChild(i);
+
+ final String identifier = declarationTree.getChild(0).getText();
+ final Tree requiredOrOptionalTree = declarationTree.getChild(1);
+ final boolean required = requiredOrOptionalTree.getType() == REQUIRED;
+
+ final String segmentName = declarationTree.getChild(2).getText();
+
+ final Declaration declaration = new Declaration() {
+ @Override
+ public String getAlias() {
+ return identifier;
+ }
+
+ @Override
+ public boolean isRequired() {
+ return required;
+ }
+
+ @Override
+ public Object getDeclaredValue(final HL7Message message) {
+ if (message == null) {
+ return null;
+ }
+
+ return message.getSegments(segmentName);
+ }
+ };
+
+ declarations.add(declaration);
+ }
+ }
+
+ private List<Selection> processSelect(final Tree select) {
+ final List<Selection> selections = new ArrayList<>();
+
+ for (int i = 0; i < select.getChildCount(); i++) {
+ final Tree selectable = select.getChild(i);
+
+ final String alias = getSelectedName(selectable);
+ final Evaluator<?> selectionEvaluator = buildReferenceEvaluator(selectable);
+ final Selection selection = new Selection(selectionEvaluator, alias);
+ selections.add(selection);
+ }
+
+ return selections;
+ }
+
+ private String getSelectedName(final Tree selectable) {
+ if (selectable.getChildCount() == 0) {
+ return selectable.getText();
+ } else if (selectable.getType() == DOT) {
+ return getSelectedName(selectable.getChild(0)) + "." + getSelectedName(selectable.getChild(1));
+ } else {
+ return selectable.getChild(selectable.getChildCount() - 1).getText();
+ }
+ }
+
+ private BooleanEvaluator processWhere(final Tree where) {
+ return buildBooleanEvaluator(where.getChild(0));
+ }
+
+ private Evaluator<?> buildReferenceEvaluator(final Tree tree) {
+ switch (tree.getType()) {
+ case MESSAGE:
+ return new MessageEvaluator();
+ case SEGMENT_NAME:
+ return new SegmentEvaluator(new StringLiteralEvaluator(tree.getText()));
+ case IDENTIFIER:
+ return new DeclaredReferenceEvaluator(new StringLiteralEvaluator(tree.getText()));
+ case DOT:
+ final Tree firstChild = tree.getChild(0);
+ final Tree secondChild = tree.getChild(1);
+ return new DotEvaluator(buildReferenceEvaluator(firstChild), buildIntegerEvaluator(secondChild));
+ case STRING_LITERAL:
+ return new StringLiteralEvaluator(tree.getText());
+ case NUMBER:
+ return new IntegerLiteralEvaluator(Integer.parseInt(tree.getText()));
+ default:
+ throw new HL7QueryParsingException("Failed to build evaluator for " + tree.getText());
+ }
+ }
+
+ private IntegerEvaluator buildIntegerEvaluator(final Tree tree) {
+ switch (tree.getType()) {
+ case NUMBER:
+ return new IntegerLiteralEvaluator(Integer.parseInt(tree.getText()));
+ default:
+ throw new HL7QueryParsingException("Failed to build Integer Evaluator for " + tree.getText());
+ }
+ }
+
+ private BooleanEvaluator buildBooleanEvaluator(final Tree tree) {
+ // TODO: add Date comparisons
+ // LT/GT/GE/GE should allow for dates based on Field's Type
+ // BETWEEN
+ // DATE('2015/01/01')
+ // DATE('2015/01/01 12:00:00')
+ // DATE('24 HOURS AGO')
+ // DATE('YESTERDAY')
+
+ switch (tree.getType()) {
+ case EQUALS:
+ return new EqualsEvaluator(buildReferenceEvaluator(tree.getChild(0)), buildReferenceEvaluator(tree.getChild(1)));
+ case NOT_EQUALS:
+ return new NotEqualsEvaluator(buildReferenceEvaluator(tree.getChild(0)), buildReferenceEvaluator(tree.getChild(1)));
+ case GT:
+ return new GreaterThanEvaluator(buildReferenceEvaluator(tree.getChild(0)), buildReferenceEvaluator(tree.getChild(1)));
+ case LT:
+ return new LessThanEvaluator(buildReferenceEvaluator(tree.getChild(0)), buildReferenceEvaluator(tree.getChild(1)));
+ case GE:
+ return new GreaterThanOrEqualEvaluator(buildReferenceEvaluator(tree.getChild(0)), buildReferenceEvaluator(tree.getChild(1)));
+ case LE:
+ return new LessThanOrEqualEvaluator(buildReferenceEvaluator(tree.getChild(0)), buildReferenceEvaluator(tree.getChild(1)));
+ case NOT:
+ return new NotEvaluator(buildBooleanEvaluator(tree.getChild(0)));
+ case AND:
+ return new AndEvaluator(buildBooleanEvaluator(tree.getChild(0)), buildBooleanEvaluator(tree.getChild(1)));
+ case OR:
+ return new OrEvaluator(buildBooleanEvaluator(tree.getChild(0)), buildBooleanEvaluator(tree.getChild(1)));
+ case IS_NULL:
+ return new IsNullEvaluator(buildReferenceEvaluator(tree.getChild(0)));
+ case NOT_NULL:
+ return new NotNullEvaluator(buildReferenceEvaluator(tree.getChild(0)));
+ default:
+ throw new HL7QueryParsingException("Cannot build boolean evaluator for '" + tree.getText() + "'");
+ }
+ }
+
+ Tree getTree() {
+ return tree;
+ }
+
+ public String getQuery() {
+ return query;
+ }
+
+ @Override
+ public String toString() {
+ return "HL7Query[" + query + "]";
+ }
+
+ public static HL7Query compile(final String query) {
+ try {
+ final CommonTokenStream lexerTokenStream = createTokenStream(query);
+ final HL7QueryParser parser = new HL7QueryParser(lexerTokenStream);
+ final Tree tree = (Tree) parser.query().getTree();
+
+ return new HL7Query(tree, query);
+ } catch (final HL7QueryParsingException e) {
+ throw e;
+ } catch (final Exception e) {
+ throw new HL7QueryParsingException(e);
+ }
+ }
+
+ private static CommonTokenStream createTokenStream(final String expression) throws HL7QueryParsingException {
+ final CharStream input = new ANTLRStringStream(expression);
+ final HL7QueryLexer lexer = new HL7QueryLexer(input);
+ return new CommonTokenStream(lexer);
+ }
+
+ public List<Class<?>> getReturnTypes() {
+ final List<Class<?>> returnTypes = new ArrayList<>();
+
+ for (final Selection selection : selections) {
+ returnTypes.add(selection.getEvaluator().getType());
+ }
+
+ return returnTypes;
+ }
+
+ @SuppressWarnings("unchecked")
+ public QueryResult evaluate(final HL7Message message) {
+
+ int totalIterations = 1;
+ final LinkedHashMap<String, List<Object>> possibleValueMap = new LinkedHashMap<>();
+ for (final Declaration declaration : declarations) {
+ final Object value = declaration.getDeclaredValue(message);
+ if (value == null && declaration.isRequired()) {
+ return new MissedResult(selections);
+ }
+
+ final List<Object> possibleValues;
+ if (value instanceof List) {
+ possibleValues = (List<Object>) value;
+ } else if (value instanceof Collection) {
+ possibleValues = new ArrayList<Object>((Collection<Object>) value);
+ } else {
+ possibleValues = new ArrayList<>(1);
+ possibleValues.add(value);
+ }
+
+ if (possibleValues.isEmpty()) {
+ return new MissedResult(selections);
+ }
+
+ possibleValueMap.put(declaration.getAlias(), possibleValues);
+ totalIterations *= possibleValues.size();
+ }
+
+ final Set<Map<String, Object>> resultSet = new HashSet<>();
+ for (int i = 0; i < totalIterations; i++) {
+ final Map<String, Object> aliasValues = assignAliases(possibleValueMap, i);
+
+ aliasValues.put(Evaluator.MESSAGE_KEY, message);
+ if (whereEvaluator == null || Boolean.TRUE.equals(whereEvaluator.evaluate(aliasValues))) {
+ final Map<String, Object> resultMap = new HashMap<>();
+
+ for (final Selection selection : selections) {
+ final Object value = selection.getEvaluator().evaluate(aliasValues);
+ resultMap.put(selection.getName(), value);
+ }
+
+ resultSet.add(resultMap);
+ }
+ }
+
+ return new StandardQueryResult(selections, resultSet);
+ }
+
+ /**
+ * assigns one of the possible values to each alias, based on which iteration this is.
+ * require LinkedHashMap just to be very clear and explicit that the order of the Map MUST be guaranteed
+ * between multiple invocations of this method.
+ * package protected for testing visibility
+ */
+ static Map<String, Object> assignAliases(final LinkedHashMap<String, List<Object>> possibleValues, final int iteration) {
+ final Map<String, Object> aliasMap = new HashMap<>();
+
+ int divisor = 1;
+ for (final Map.Entry<String, List<Object>> entry : possibleValues.entrySet()) {
+ final String alias = entry.getKey();
+ final List<Object> validValues = entry.getValue();
+
+ final int idx = (iteration / divisor) % validValues.size();
+ final Object obj = validValues.get(idx);
+ aliasMap.put(alias, obj);
+
+ divisor *= validValues.size();
+ }
+
+ return aliasMap;
+ }
+
+ public String toTreeString() {
+ final StringBuilder sb = new StringBuilder();
+ toTreeString(tree, sb, 0);
+ return sb.toString();
+ }
+
+ private void toTreeString(final Tree tree, final StringBuilder sb, final int indentLevel) {
+ final String nodeName = tree.getText();
+ for (int i = 0; i < indentLevel; i++) {
+ sb.append(" ");
+ }
+ sb.append(nodeName);
+ sb.append("\n");
+
+ for (int i = 0; i < tree.getChildCount(); i++) {
+ final Tree child = tree.getChild(i);
+ toTreeString(child, sb, indentLevel + 2);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/QueryResult.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/QueryResult.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/QueryResult.java
new file mode 100644
index 0000000..033ac0a
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/QueryResult.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.query;
+
+import java.util.List;
+
+public interface QueryResult {
+
+ boolean isMatch();
+
+ List<String> getLabels();
+
+ int getHitCount();
+
+ ResultHit nextHit();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/ResultHit.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/ResultHit.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/ResultHit.java
new file mode 100644
index 0000000..9eb1d8f
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/ResultHit.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.query;
+
+import java.util.Map;
+
+public interface ResultHit {
+
+ Object getValue(String label);
+
+ Map<String, Object> getSelectedValues();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/Selection.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/Selection.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/Selection.java
new file mode 100644
index 0000000..7b5b7b3
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/Selection.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.query;
+
+import org.apache.nifi.hl7.query.evaluator.Evaluator;
+
+public class Selection {
+
+ private final Evaluator<?> evaluator;
+ private final String name;
+
+ public Selection(final Evaluator<?> evaluator, final String name) {
+ this.evaluator = evaluator;
+ this.name = name;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public Evaluator<?> getEvaluator() {
+ return evaluator;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/BooleanEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/BooleanEvaluator.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/BooleanEvaluator.java
new file mode 100644
index 0000000..98b077e
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/BooleanEvaluator.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.query.evaluator;
+
+public abstract class BooleanEvaluator implements Evaluator<Boolean> {
+
+ public Class<? extends Boolean> getType() {
+ return Boolean.class;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/Evaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/Evaluator.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/Evaluator.java
new file mode 100644
index 0000000..2d1ddc7
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/Evaluator.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.query.evaluator;
+
+import java.util.Map;
+
+public interface Evaluator<T> {
+
+ public static final String MESSAGE_KEY = "message";
+
+ T evaluate(Map<String, Object> objectMap);
+
+ Class<? extends T> getType();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/IntegerEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/IntegerEvaluator.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/IntegerEvaluator.java
new file mode 100644
index 0000000..ed56d80
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/IntegerEvaluator.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.query.evaluator;
+
+public abstract class IntegerEvaluator implements Evaluator<Integer> {
+
+ public Class<? extends Integer> getType() {
+ return Integer.class;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/StringEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/StringEvaluator.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/StringEvaluator.java
new file mode 100644
index 0000000..53ab004
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/StringEvaluator.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.query.evaluator;
+
+public abstract class StringEvaluator implements Evaluator<String> {
+
+ public Class<? extends String> getType() {
+ return String.class;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/AbstractComparisonEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/AbstractComparisonEvaluator.java b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/AbstractComparisonEvaluator.java
new file mode 100644
index 0000000..45edf64
--- /dev/null
+++ b/nifi-commons/nifi-hl7-query-language/src/main/java/org/apache/nifi/hl7/query/evaluator/comparison/AbstractComparisonEvaluator.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hl7.query.evaluator.comparison;
+
+import java.util.Collection;
+import java.util.Map;
+
+import org.apache.nifi.hl7.model.HL7Field;
+import org.apache.nifi.hl7.query.evaluator.BooleanEvaluator;
+import org.apache.nifi.hl7.query.evaluator.Evaluator;
+
+public abstract class AbstractComparisonEvaluator extends BooleanEvaluator {
+
+ private final Evaluator<?> lhs;
+ private final Evaluator<?> rhs;
+
+ public AbstractComparisonEvaluator(final Evaluator<?> lhs, final Evaluator<?> rhs) {
+ this.lhs = lhs;
+ this.rhs = rhs;
+ }
+
+ @Override
+ public final Boolean evaluate(final Map<String, Object> objectMap) {
+ final Object lhsValue = lhs.evaluate(objectMap);
+ if (lhsValue == null) {
+ return false;
+ }
+
+ final Object rhsValue = rhs.evaluate(objectMap);
+ if (rhsValue == null) {
+ return false;
+ }
+
+ return compareRaw(lhsValue, rhsValue);
+ }
+
+ private Boolean compareRaw(Object lhsValue, Object rhsValue) {
+ if (lhsValue == null || rhsValue == null) {
+ return false;
+ }
+
+ if (lhsValue instanceof HL7Field) {
+ lhsValue = ((HL7Field) lhsValue).getValue();
+ }
+
+ if (rhsValue instanceof HL7Field) {
+ rhsValue = ((HL7Field) rhsValue).getValue();
+ }
+
+ if (lhsValue == null || rhsValue == null) {
+ return false;
+ }
+
+ /**
+ * both are collections, and compare(lhsValue, rhsValue) is false.
+ * this would be the case, for instance, if we compared field 1 of one segment to
+ * a field in another segment, and both fields had components.
+ */
+ if (lhsValue instanceof Collection && rhsValue instanceof Collection) {
+ return false;
+ }
+
+ /**
+ * if one side is a collection but the other is not, check if any element in that
+ * collection compares to the other element in a way that satisfies the condition.
+ * this would happen, for instance, if we check Segment1.Field5 = 'X' and field 5 repeats
+ * with a value "A~B~C~X~Y~Z"; in this case we do want to consider Field 5 = X as true.
+ */
+ if (lhsValue instanceof Collection) {
+ for (final Object lhsObject : (Collection<?>) lhsValue) {
+ if (compareRaw(lhsObject, rhsValue)) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ if (rhsValue instanceof Collection) {
+ for (final Object rhsObject : (Collection<?>) rhsValue) {
+ if (compareRaw(rhsObject, lhsValue)) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ if (lhsValue != null && rhsValue != null && compare(lhsValue, rhsValue)) {
+ return true;
+ }
+
+ return false;
+ }
+
+ protected abstract boolean compare(Object lhs, Object rhs);
+}
[05/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/impl/StandardUserDAO.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/impl/StandardUserDAO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/impl/StandardUserDAO.java
new file mode 100644
index 0000000..ea7c1a1
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/impl/StandardUserDAO.java
@@ -0,0 +1,634 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.dao.impl;
+
+import java.nio.charset.StandardCharsets;
+import java.sql.Connection;
+import org.apache.nifi.admin.dao.UserDAO;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.Types;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.UUID;
+import org.apache.nifi.admin.RepositoryUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.admin.dao.DataAccessException;
+import org.apache.nifi.authorization.Authority;
+import org.apache.nifi.user.AccountStatus;
+import org.apache.nifi.user.NiFiUser;
+
+/**
+ * Responsible for loading and persisting NiFiUsers.
+ */
+public class StandardUserDAO implements UserDAO {
+
+ private static final String SELECT_PENDING_ACCOUNTS_COUNT = "SELECT "
+ + "COUNT(*) as PENDING_ACCOUNTS "
+ + "FROM USER U "
+ + "WHERE U.STATUS = 'PENDING'";
+
+ private static final String SELECT_USER_BY_DN = "SELECT "
+ + "U.ID, "
+ + "U.DN, "
+ + "U.USER_NAME, "
+ + "U.USER_GROUP, "
+ + "U.CREATION, "
+ + "U.LAST_ACCESSED, "
+ + "U.LAST_VERIFIED, "
+ + "U.JUSTIFICATION, "
+ + "U.STATUS, "
+ + "A.ROLE "
+ + "FROM USER U "
+ + "LEFT JOIN AUTHORITY A " // ensures that users without authorities are still matched
+ + "ON U.ID = A.USER_ID "
+ + "WHERE U.DN = ?";
+
+ private static final String SELECT_USER_BY_ID = "SELECT "
+ + "U.ID, "
+ + "U.DN, "
+ + "U.USER_NAME, "
+ + "U.USER_GROUP, "
+ + "U.CREATION, "
+ + "U.LAST_ACCESSED, "
+ + "U.LAST_VERIFIED, "
+ + "U.JUSTIFICATION, "
+ + "U.STATUS, "
+ + "A.ROLE "
+ + "FROM USER U "
+ + "LEFT JOIN AUTHORITY A " // ensures that users without authorities are still matched
+ + "ON U.ID = A.USER_ID "
+ + "WHERE U.ID = ?";
+
+ private static final String SELECT_USERS = "SELECT "
+ + "U.ID, "
+ + "U.DN, "
+ + "U.USER_NAME, "
+ + "U.USER_GROUP, "
+ + "U.CREATION, "
+ + "U.LAST_ACCESSED, "
+ + "U.LAST_VERIFIED, "
+ + "U.JUSTIFICATION, "
+ + "U.STATUS, "
+ + "A.ROLE "
+ + "FROM USER U "
+ + "LEFT JOIN AUTHORITY A " // ensures that users without authorities are still matched
+ + "ON U.ID = A.USER_ID "
+ + "WHERE U.DN <> ?";
+
+ private static final String SELECT_USER_GROUPS = "SELECT DISTINCT "
+ + "U.USER_GROUP "
+ + "FROM USER U";
+
+ private static final String SELECT_USER_GROUP = "SELECT "
+ + "U.ID, "
+ + "U.DN, "
+ + "U.USER_NAME, "
+ + "U.USER_GROUP, "
+ + "U.CREATION, "
+ + "U.LAST_ACCESSED, "
+ + "U.LAST_VERIFIED, "
+ + "U.JUSTIFICATION, "
+ + "U.STATUS, "
+ + "A.ROLE "
+ + "FROM USER U "
+ + "LEFT JOIN AUTHORITY A " // ensures that users without authorities are still matched
+ + "ON U.ID = A.USER_ID "
+ + "WHERE U.DN <> ? AND U.USER_GROUP = ?";
+
+ private static final String INSERT_USER = "INSERT INTO USER ("
+ + "ID, DN, USER_NAME, USER_GROUP, CREATION, LAST_VERIFIED, JUSTIFICATION, STATUS"
+ + ") VALUES ("
+ + "?, "
+ + "?, "
+ + "?, "
+ + "?, "
+ + "NOW(), "
+ + "?, "
+ + "?, "
+ + "?"
+ + ")";
+
+ private static final String UPDATE_USER = "UPDATE USER SET "
+ + "DN = ?, "
+ + "USER_NAME = ?, "
+ + "USER_GROUP = ?, "
+ + "LAST_ACCESSED = ?, "
+ + "LAST_VERIFIED = ?, "
+ + "JUSTIFICATION = ?, "
+ + "STATUS = ? "
+ + "WHERE ID = ?";
+
+ private static final String UPDATE_USER_GROUP_STATUS = "UPDATE USER SET "
+ + "STATUS = ?,"
+ + "USER_GROUP = NULL "
+ + "WHERE USER_GROUP = ?";
+
+ private static final String UPDATE_USER_GROUP_VERIFICATION = "UPDATE USER SET "
+ + "LAST_VERIFIED = ? "
+ + "WHERE USER_GROUP = ?";
+
+ private static final String UNGROUP_GROUP = "UPDATE USER SET "
+ + "USER_GROUP = NULL "
+ + "WHERE USER_GROUP = ?";
+
+ private static final String DELETE_USER = "DELETE FROM USER "
+ + "WHERE ID = ?";
+
+ private final Connection connection;
+
+ public StandardUserDAO(Connection connection) {
+ this.connection = connection;
+ }
+
+ @Override
+ public Boolean hasPendingUserAccounts() throws DataAccessException {
+ PreparedStatement statement = null;
+ ResultSet rs = null;
+ try {
+ // create the connection and obtain a statement
+ statement = connection.prepareStatement(SELECT_PENDING_ACCOUNTS_COUNT);
+
+ // execute the query
+ rs = statement.executeQuery();
+
+ // get the first row which will contain the number of pending accounts
+ if (rs.next()) {
+ int pendingAccounts = rs.getInt("PENDING_ACCOUNTS");
+ return pendingAccounts > 0;
+ }
+
+ // query returned no results?
+ return false;
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } finally {
+ RepositoryUtils.closeQuietly(rs);
+ RepositoryUtils.closeQuietly(statement);
+ }
+ }
+
+ @Override
+ public Set<NiFiUser> findUsers() throws DataAccessException {
+ Set<NiFiUser> users = new HashSet<>();
+
+ PreparedStatement statement = null;
+ ResultSet rs = null;
+ try {
+ // create the connection and obtain a statement
+ statement = connection.prepareStatement(SELECT_USERS);
+ statement.setString(1, NiFiUser.ANONYMOUS_USER_DN);
+
+ // execute the query
+ rs = statement.executeQuery();
+
+ // create the user
+ NiFiUser user = null;
+
+ // go through the user and its roles
+ while (rs.next()) {
+ // get the user id for the current record
+ String userId = rs.getString("ID");
+
+ // create the user during the first iteration
+ if (user == null || !userId.equals(user.getId())) {
+ user = new NiFiUser();
+ user.setId(userId);
+ user.setDn(rs.getString("DN"));
+ user.setUserName(rs.getString("USER_NAME"));
+ user.setUserGroup(rs.getString("USER_GROUP"));
+ user.setJustification(rs.getString("JUSTIFICATION"));
+ user.setStatus(AccountStatus.valueOfStatus(rs.getString("STATUS")));
+
+ // set the creation date
+ user.setCreation(new Date(rs.getTimestamp("CREATION").getTime()));
+
+ // get the last accessed date
+ if (rs.getTimestamp("LAST_ACCESSED") != null) {
+ user.setLastAccessed(new Date(rs.getTimestamp("LAST_ACCESSED").getTime()));
+ }
+
+ // get the last verified date
+ if (rs.getTimestamp("LAST_VERIFIED") != null) {
+ user.setLastVerified(new Date(rs.getTimestamp("LAST_VERIFIED").getTime()));
+ }
+
+ // add the user
+ users.add(user);
+ }
+
+ // the select statement performs a left join since the desired
+ // user may not have any authorities
+ String authority = rs.getString("ROLE");
+ if (StringUtils.isNotBlank(authority)) {
+ user.getAuthorities().add(Authority.valueOfAuthority(authority));
+ }
+ }
+
+ return users;
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } finally {
+ RepositoryUtils.closeQuietly(rs);
+ RepositoryUtils.closeQuietly(statement);
+ }
+ }
+
+ @Override
+ public Set<String> findUserGroups() throws DataAccessException {
+ Set<String> userGroups = new HashSet<>();
+
+ PreparedStatement statement = null;
+ ResultSet rs = null;
+ try {
+ // create the connection and obtain a statement
+ statement = connection.prepareStatement(SELECT_USER_GROUPS);
+
+ // execute the query
+ rs = statement.executeQuery();
+
+ // get each user group
+ while (rs.next()) {
+ userGroups.add(rs.getString("USER_GROUP"));
+ }
+
+ return userGroups;
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } finally {
+ RepositoryUtils.closeQuietly(rs);
+ RepositoryUtils.closeQuietly(statement);
+ }
+ }
+
+ @Override
+ public Set<NiFiUser> findUsersForGroup(String group) throws DataAccessException {
+ Set<NiFiUser> users = new HashSet<>();
+
+ PreparedStatement statement = null;
+ ResultSet rs = null;
+ try {
+ // create the connection and obtain a statement
+ statement = connection.prepareStatement(SELECT_USER_GROUP);
+ statement.setString(1, NiFiUser.ANONYMOUS_USER_DN);
+ statement.setString(2, group);
+
+ // execute the query
+ rs = statement.executeQuery();
+
+ // create the user
+ NiFiUser user = null;
+
+ // go through the user and its roles
+ while (rs.next()) {
+ // get the user id for the current record
+ String userId = rs.getString("ID");
+
+ // create the user during the first iteration
+ if (user == null || !userId.equals(user.getId())) {
+ user = new NiFiUser();
+ user.setId(userId);
+ user.setDn(rs.getString("DN"));
+ user.setUserName(rs.getString("USER_NAME"));
+ user.setUserGroup(rs.getString("USER_GROUP"));
+ user.setJustification(rs.getString("JUSTIFICATION"));
+ user.setStatus(AccountStatus.valueOfStatus(rs.getString("STATUS")));
+
+ // set the creation date
+ user.setCreation(new Date(rs.getTimestamp("CREATION").getTime()));
+
+ // get the last accessed date
+ if (rs.getTimestamp("LAST_ACCESSED") != null) {
+ user.setLastAccessed(new Date(rs.getTimestamp("LAST_ACCESSED").getTime()));
+ }
+
+ // get the last verified date
+ if (rs.getTimestamp("LAST_VERIFIED") != null) {
+ user.setLastVerified(new Date(rs.getTimestamp("LAST_VERIFIED").getTime()));
+ }
+
+ // add the user
+ users.add(user);
+ }
+
+ // the select statement performs a left join since the desired
+ // user may not have any authorities
+ String authority = rs.getString("ROLE");
+ if (StringUtils.isNotBlank(authority)) {
+ user.getAuthorities().add(Authority.valueOfAuthority(authority));
+ }
+ }
+
+ return users;
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } finally {
+ RepositoryUtils.closeQuietly(rs);
+ RepositoryUtils.closeQuietly(statement);
+ }
+ }
+
+ @Override
+ public NiFiUser findUserById(String id) throws DataAccessException {
+ PreparedStatement statement = null;
+ ResultSet rs = null;
+ try {
+ // create the connection and obtain a statement
+ statement = connection.prepareStatement(SELECT_USER_BY_ID);
+ statement.setString(1, id);
+
+ // execute the query
+ rs = statement.executeQuery();
+
+ // create the user
+ NiFiUser user = null;
+
+ // go through the user and its roles
+ while (rs.next()) {
+ // create the user during the first iteration
+ if (user == null) {
+ user = new NiFiUser();
+ user.setId(rs.getString("ID"));
+ user.setDn(rs.getString("DN"));
+ user.setUserName(rs.getString("USER_NAME"));
+ user.setUserGroup(rs.getString("USER_GROUP"));
+ user.setJustification(rs.getString("JUSTIFICATION"));
+ user.setStatus(AccountStatus.valueOfStatus(rs.getString("STATUS")));
+
+ // set the creation date
+ user.setCreation(new Date(rs.getTimestamp("CREATION").getTime()));
+
+ // get the last accessed date
+ if (rs.getTimestamp("LAST_ACCESSED") != null) {
+ user.setLastAccessed(new Date(rs.getTimestamp("LAST_ACCESSED").getTime()));
+ }
+
+ // get the last verified date
+ if (rs.getTimestamp("LAST_VERIFIED") != null) {
+ user.setLastVerified(new Date(rs.getTimestamp("LAST_VERIFIED").getTime()));
+ }
+ }
+
+ // the select statement performs a left join since the desired
+ // user may not have any authorities
+ String authority = rs.getString("ROLE");
+ if (StringUtils.isNotBlank(authority)) {
+ user.getAuthorities().add(Authority.valueOfAuthority(authority));
+ }
+ }
+
+ return user;
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } finally {
+ RepositoryUtils.closeQuietly(rs);
+ RepositoryUtils.closeQuietly(statement);
+ }
+ }
+
+ @Override
+ public NiFiUser findUserByDn(String dn) throws DataAccessException {
+ PreparedStatement statement = null;
+ ResultSet rs = null;
+ try {
+ // create the connection and obtain a statement
+ statement = connection.prepareStatement(SELECT_USER_BY_DN);
+ statement.setString(1, dn);
+
+ // execute the query
+ rs = statement.executeQuery();
+
+ // create the user
+ NiFiUser user = null;
+
+ // go through the user and its roles
+ while (rs.next()) {
+ // create the user during the first iteration
+ if (user == null) {
+ user = new NiFiUser();
+ user.setId(rs.getString("ID"));
+ user.setDn(rs.getString("DN"));
+ user.setUserName(rs.getString("USER_NAME"));
+ user.setUserGroup(rs.getString("USER_GROUP"));
+ user.setJustification(rs.getString("JUSTIFICATION"));
+ user.setStatus(AccountStatus.valueOfStatus(rs.getString("STATUS")));
+
+ // set the creation date
+ user.setCreation(new Date(rs.getTimestamp("CREATION").getTime()));
+
+ // get the last accessed date
+ if (rs.getTimestamp("LAST_ACCESSED") != null) {
+ user.setLastAccessed(new Date(rs.getTimestamp("LAST_ACCESSED").getTime()));
+ }
+
+ // get the last verified date
+ if (rs.getTimestamp("LAST_VERIFIED") != null) {
+ user.setLastVerified(new Date(rs.getTimestamp("LAST_VERIFIED").getTime()));
+ }
+ }
+
+ // the select statement performs a left join since the desired
+ // user may not have any authorities
+ String authority = rs.getString("ROLE");
+ if (StringUtils.isNotBlank(authority)) {
+ user.getAuthorities().add(Authority.valueOfAuthority(authority));
+ }
+ }
+
+ return user;
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } finally {
+ RepositoryUtils.closeQuietly(rs);
+ RepositoryUtils.closeQuietly(statement);
+ }
+ }
+
+ @Override
+ public void createUser(NiFiUser user) throws DataAccessException {
+ if (user.getDn() == null) {
+ throw new IllegalArgumentException("User dn must be specified.");
+ }
+
+ PreparedStatement statement = null;
+ ResultSet rs = null;
+ try {
+ final String id = UUID.nameUUIDFromBytes(user.getDn().getBytes(StandardCharsets.UTF_8)).toString();
+
+ // create a statement
+ statement = connection.prepareStatement(INSERT_USER, Statement.RETURN_GENERATED_KEYS);
+ statement.setString(1, id);
+ statement.setString(2, StringUtils.left(user.getDn(), 255));
+ statement.setString(3, StringUtils.left(user.getUserName(), 100));
+ statement.setString(4, StringUtils.left(user.getUserGroup(), 100));
+ if (user.getLastVerified() != null) {
+ statement.setTimestamp(5, new java.sql.Timestamp(user.getLastVerified().getTime()));
+ } else {
+ statement.setTimestamp(5, null);
+ }
+ statement.setString(6, StringUtils.left(user.getJustification(), 500));
+ statement.setString(7, user.getStatus().toString());
+
+ // insert the user
+ int updateCount = statement.executeUpdate();
+ if (updateCount == 1) {
+ user.setId(id);
+ } else {
+ throw new DataAccessException("Unable to insert user.");
+ }
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } catch (DataAccessException dae) {
+ throw dae;
+ } finally {
+ RepositoryUtils.closeQuietly(rs);
+ RepositoryUtils.closeQuietly(statement);
+ }
+ }
+
+ @Override
+ public void deleteUser(String id) throws DataAccessException {
+ // ensure there are some authorities to create
+ PreparedStatement statement = null;
+ try {
+ // add each authority for the specified user
+ statement = connection.prepareStatement(DELETE_USER);
+ statement.setString(1, id);
+
+ // insert the authorities
+ statement.executeUpdate();
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } catch (DataAccessException dae) {
+ throw dae;
+ } finally {
+ RepositoryUtils.closeQuietly(statement);
+ }
+ }
+
+ @Override
+ public void updateUser(NiFiUser user) throws DataAccessException {
+ PreparedStatement statement = null;
+ try {
+ // create a statement
+ statement = connection.prepareStatement(UPDATE_USER);
+ statement.setString(1, StringUtils.left(user.getDn(), 255));
+ statement.setString(2, StringUtils.left(user.getUserName(), 100));
+ statement.setString(3, StringUtils.left(user.getUserGroup(), 100));
+ statement.setString(6, StringUtils.left(user.getJustification(), 500));
+ statement.setString(7, user.getStatus().toString());
+ statement.setString(8, user.getId());
+
+ // set the last accessed time accordingly
+ if (user.getLastAccessed() == null) {
+ statement.setNull(4, Types.TIMESTAMP);
+ } else {
+ statement.setTimestamp(4, new java.sql.Timestamp(user.getLastAccessed().getTime()));
+ }
+
+ // set the last verified time accordingly
+ if (user.getLastVerified() == null) {
+ statement.setNull(5, Types.TIMESTAMP);
+ } else {
+ statement.setTimestamp(5, new java.sql.Timestamp(user.getLastVerified().getTime()));
+ }
+
+ // perform the update
+ int updateCount = statement.executeUpdate();
+ if (updateCount != 1) {
+ throw new DataAccessException("Unable to update user.");
+ }
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } catch (DataAccessException dae) {
+ throw dae;
+ } finally {
+ RepositoryUtils.closeQuietly(statement);
+ }
+ }
+
+ @Override
+ public void updateGroupStatus(String group, AccountStatus status) throws DataAccessException {
+ PreparedStatement statement = null;
+ try {
+ // create a statement
+ statement = connection.prepareStatement(UPDATE_USER_GROUP_STATUS);
+ statement.setString(1, status.toString());
+ statement.setString(2, group);
+
+ // perform the update
+ statement.executeUpdate();
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } catch (DataAccessException dae) {
+ throw dae;
+ } finally {
+ RepositoryUtils.closeQuietly(statement);
+ }
+ }
+
+ @Override
+ public void updateGroupVerification(String group, Date lastVerified) throws DataAccessException {
+ PreparedStatement statement = null;
+ try {
+ // create a statement
+ statement = connection.prepareStatement(UPDATE_USER_GROUP_VERIFICATION);
+
+ // set the last verified time accordingly
+ if (lastVerified == null) {
+ statement.setNull(1, Types.TIMESTAMP);
+ } else {
+ statement.setTimestamp(1, new java.sql.Timestamp(lastVerified.getTime()));
+ }
+
+ // set the group
+ statement.setString(2, group);
+
+ // perform the update
+ statement.executeUpdate();
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } catch (DataAccessException dae) {
+ throw dae;
+ } finally {
+ RepositoryUtils.closeQuietly(statement);
+ }
+ }
+
+ @Override
+ public void ungroup(String group) throws DataAccessException {
+ PreparedStatement statement = null;
+ try {
+ // create a statement
+ statement = connection.prepareStatement(UNGROUP_GROUP);
+ statement.setString(1, group);
+
+ // perform the update
+ statement.executeUpdate();
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } catch (DataAccessException dae) {
+ throw dae;
+ } finally {
+ RepositoryUtils.closeQuietly(statement);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/AccountDisabledException.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/AccountDisabledException.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/AccountDisabledException.java
new file mode 100644
index 0000000..e8b3d10
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/AccountDisabledException.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service;
+
+/**
+ * Exception to indicate that the user account is disabled.
+ */
+public class AccountDisabledException extends RuntimeException {
+
+ public AccountDisabledException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ }
+
+ public AccountDisabledException(Throwable cause) {
+ super(cause);
+ }
+
+ public AccountDisabledException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public AccountDisabledException(String message) {
+ super(message);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/AccountNotFoundException.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/AccountNotFoundException.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/AccountNotFoundException.java
new file mode 100644
index 0000000..88287ce
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/AccountNotFoundException.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service;
+
+/**
+ * Exception to indicate that the user account is disabled.
+ */
+public class AccountNotFoundException extends RuntimeException {
+
+ public AccountNotFoundException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ }
+
+ public AccountNotFoundException(Throwable cause) {
+ super(cause);
+ }
+
+ public AccountNotFoundException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public AccountNotFoundException(String message) {
+ super(message);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/AccountPendingException.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/AccountPendingException.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/AccountPendingException.java
new file mode 100644
index 0000000..dacc483
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/AccountPendingException.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service;
+
+/**
+ * Exception to indicate that the user has already submitting an account request
+ * and that request is still pending.
+ */
+public class AccountPendingException extends RuntimeException {
+
+ public AccountPendingException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ }
+
+ public AccountPendingException(Throwable cause) {
+ super(cause);
+ }
+
+ public AccountPendingException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public AccountPendingException(String message) {
+ super(message);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/AdministrationException.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/AdministrationException.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/AdministrationException.java
new file mode 100644
index 0000000..c0e8ac1
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/AdministrationException.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service;
+
+/**
+ *
+ */
+public class AdministrationException extends RuntimeException {
+
+ public AdministrationException(Throwable cause) {
+ super(cause);
+ }
+
+ public AdministrationException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public AdministrationException(String message) {
+ super(message);
+ }
+
+ public AdministrationException() {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/AuditService.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/AuditService.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/AuditService.java
new file mode 100644
index 0000000..680173f
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/AuditService.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service;
+
+import java.util.Collection;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import org.apache.nifi.action.Action;
+import org.apache.nifi.history.HistoryQuery;
+import org.apache.nifi.history.History;
+import org.apache.nifi.history.PreviousValue;
+
+/**
+ * Allows NiFi actions to be audited.
+ */
+public interface AuditService {
+
+ /**
+ * Adds the specified actions.
+ *
+ * @param actions to add
+ * @throws AdministrationException if failed to add
+ */
+ void addActions(Collection<Action> actions);
+
+ /**
+ * @param componentId identifier of the component
+ * @return Finds the previous values for the specified property in the
+ * specified component. Returns null if there are none
+ */
+ Map<String, List<PreviousValue>> getPreviousValues(String componentId);
+
+ /**
+ * Get the actions within the given date range.
+ *
+ * @param actionQuery query
+ * @return history of actions
+ * @throws AdministrationException ae
+ */
+ History getActions(HistoryQuery actionQuery);
+
+ /**
+ * Get the details for the specified action id. If the action cannot be
+ * found, null is returned.
+ *
+ * @param actionId identifier of action
+ * @return the action
+ */
+ Action getAction(Integer actionId);
+
+ /**
+ * Purges all action's that occurred before the specified end date.
+ *
+ * @param end the stopper for event purging
+ * @param purgeAction the action
+ * @throws AdministrationException ae
+ */
+ void purgeActions(Date end, Action purgeAction);
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/UserService.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/UserService.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/UserService.java
new file mode 100644
index 0000000..b02f192
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/UserService.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.nifi.authorization.Authority;
+import org.apache.nifi.authorization.DownloadAuthorization;
+import org.apache.nifi.user.NiFiUser;
+import org.apache.nifi.user.NiFiUserGroup;
+
+/**
+ * Manages NiFi user accounts.
+ */
+public interface UserService {
+
+ /**
+ * Creates a new user account using the specified dn and justification.
+ *
+ * @param dn user dn
+ * @param justification why the account is necessary
+ * @return the created NiFiUser
+ */
+ NiFiUser createPendingUserAccount(String dn, String justification);
+
+ /**
+ * @return Determines if there are any PENDING user accounts present
+ */
+ Boolean hasPendingUserAccount();
+
+ /**
+ * @param dnChain user dn chain
+ * @param attributes attributes for authorization request
+ * @return Determines if the users in the dnChain are authorized to download
+ * content with the specified attributes
+ */
+ DownloadAuthorization authorizeDownload(List<String> dnChain, Map<String, String> attributes);
+
+ /**
+ * Updates a user group using the specified group comprised of the specified
+ * users. Returns all the users that are currently in the specified group.
+ *
+ * @param group group
+ * @param userIds users
+ * @param authorities auths
+ * @return a user group
+ */
+ NiFiUserGroup updateGroup(String group, Set<String> userIds, Set<Authority> authorities);
+
+ /**
+ * Authorizes the user specified.
+ *
+ * @param dn user dn
+ * @return the user for the given dn if found
+ */
+ NiFiUser checkAuthorization(String dn);
+
+ /**
+ * Deletes the user with the specified id.
+ *
+ * @param id user identifier
+ */
+ void deleteUser(String id);
+
+ /**
+ * Disables the specified users account.
+ *
+ * @param id user identifier
+ * @return user for the given identifier
+ */
+ NiFiUser disable(String id);
+
+ /**
+ * Disables the specified user group.
+ *
+ * @param group to disable
+ * @return user group
+ */
+ NiFiUserGroup disableGroup(String group);
+
+ /**
+ * Updates the specified user with the specified authorities.
+ *
+ * @param id identifier of user
+ * @param authorities auths to set
+ * @return the updated user
+ */
+ NiFiUser update(String id, Set<Authority> authorities);
+
+ /**
+ * Invalidates the specified user account.
+ *
+ * @param id identifier of user account to invalidate
+ */
+ void invalidateUserAccount(String id);
+
+ /**
+ * Invalidates the user accounts associated with the specified user group.
+ *
+ * @param group to invalidate user accounts on
+ */
+ void invalidateUserGroupAccount(String group);
+
+ /**
+ * Ungroups the specified group.
+ *
+ * @param group to split up
+ */
+ void ungroup(String group);
+
+ /**
+ * Ungroups the specified user.
+ *
+ * @param id user to ungroup
+ */
+ void ungroupUser(String id);
+
+ /**
+ * Returns a collection of all NiFiUsers.
+ *
+ * @return Collection of users
+ */
+ Collection<NiFiUser> getUsers();
+
+ /**
+ * Finds the specified user by id.
+ *
+ * @param id of the user
+ * @return the user object
+ */
+ NiFiUser getUserById(String id);
+
+ /**
+ * Finds the specified user by dn.
+ *
+ * @param dn the user dn
+ * @return the newly created user
+ * @throws AdministrationException ae
+ */
+ NiFiUser getUserByDn(String dn);
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/AbstractUserAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/AbstractUserAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/AbstractUserAction.java
new file mode 100644
index 0000000..b970dc1
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/AbstractUserAction.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import java.util.Date;
+import java.util.EnumSet;
+import java.util.Set;
+import org.apache.nifi.authorization.Authority;
+import org.apache.nifi.authorization.AuthorityProvider;
+import org.apache.nifi.user.AccountStatus;
+import org.apache.nifi.user.NiFiUser;
+
+/**
+ *
+ * @param <T> type of user action
+ */
+public abstract class AbstractUserAction<T> implements AdministrationAction<T> {
+
+ /**
+ * Determines the authorities that need to be added to the specified user.
+ *
+ * @param user user
+ * @param authorities auths
+ * @return authorities to add
+ */
+ protected Set<Authority> determineAuthoritiesToAdd(NiFiUser user, Set<Authority> authorities) {
+ // not using copyOf since authorities may be empty and copyOf can throw an IllegalArgumentException when empty
+ Set<Authority> authoritiesToAdd = EnumSet.noneOf(Authority.class);
+ authoritiesToAdd.addAll(authorities);
+
+ // identify the authorities that need to be inserted
+ authoritiesToAdd.removeAll(user.getAuthorities());
+
+ // return the desired authorities
+ return authoritiesToAdd;
+ }
+
+ /**
+ * Determines the authorities that need to be removed from the specified
+ * user.
+ *
+ * @param user user
+ * @param authorities auths
+ * @return auths to remove
+ */
+ protected Set<Authority> determineAuthoritiesToRemove(NiFiUser user, Set<Authority> authorities) {
+ Set<Authority> authoritiesToRemove = EnumSet.copyOf(user.getAuthorities());
+
+ // identify the authorities that need to be removed
+ authoritiesToRemove.removeAll(authorities);
+
+ // return the desired authorities
+ return authoritiesToRemove;
+ }
+
+ /**
+ * Verifies the specified users account. Includes obtaining the authorities
+ * and group according to the specified authority provider.
+ *
+ * @param authorityProvider provider
+ * @param user user to verify
+ */
+ protected void verifyAccount(AuthorityProvider authorityProvider, NiFiUser user) {
+ // load the roles for the user
+ Set<Authority> authorities = authorityProvider.getAuthorities(user.getDn());
+
+ // update the user's authorities
+ user.getAuthorities().clear();
+ user.getAuthorities().addAll(authorities);
+
+ // get the user group
+ user.setUserGroup(authorityProvider.getGroupForUser(user.getDn()));
+
+ // update the users status in case they were previously pending or disabled
+ user.setStatus(AccountStatus.ACTIVE);
+
+ // update the users last verified time - this timestampt shouldn't be record
+ // until the both the user's authorities and group have been synced
+ Date now = new Date();
+ user.setLastVerified(now);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/AddActionsAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/AddActionsAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/AddActionsAction.java
new file mode 100644
index 0000000..5a2159f
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/AddActionsAction.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import java.util.Collection;
+import org.apache.nifi.action.Action;
+import org.apache.nifi.admin.dao.ActionDAO;
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.authorization.AuthorityProvider;
+
+/**
+ * Adds the specified actions.
+ */
+public class AddActionsAction implements AdministrationAction<Void> {
+
+ private final Collection<Action> actions;
+
+ public AddActionsAction(Collection<Action> actions) {
+ this.actions = actions;
+ }
+
+ @Override
+ public Void execute(DAOFactory daoFactory, AuthorityProvider authorityProvider) {
+ ActionDAO actionDao = daoFactory.getActionDAO();
+
+ // add each action
+ for (Action action : actions) {
+ actionDao.createAction(action);
+ }
+
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/AdministrationAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/AdministrationAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/AdministrationAction.java
new file mode 100644
index 0000000..f1795a9
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/AdministrationAction.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.authorization.AuthorityProvider;
+
+/**
+ * Defines the administration action. Actions are provided a DAO factory and
+ * authority provider to perform a require action.
+ *
+ * @param <T> type
+ */
+public interface AdministrationAction<T> {
+
+ /**
+ * Performs an action using the specified DAOFactory and AuthorityProvider.
+ *
+ * @param daoFactory factory
+ * @param authorityProvider provider
+ * @return action result
+ */
+ T execute(DAOFactory daoFactory, AuthorityProvider authorityProvider);
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/AuthorizeDownloadAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/AuthorizeDownloadAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/AuthorizeDownloadAction.java
new file mode 100644
index 0000000..d1b994c
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/AuthorizeDownloadAction.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import java.util.List;
+import java.util.Map;
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.admin.service.AccountNotFoundException;
+import org.apache.nifi.admin.service.AdministrationException;
+import org.apache.nifi.authorization.AuthorityProvider;
+import org.apache.nifi.authorization.DownloadAuthorization;
+import org.apache.nifi.authorization.exception.AuthorityAccessException;
+import org.apache.nifi.authorization.exception.UnknownIdentityException;
+
+/**
+ * Attempts to obtain authorization to download the content with the specified
+ * attributes for the specified user.
+ */
+public class AuthorizeDownloadAction implements AdministrationAction<DownloadAuthorization> {
+
+ private final List<String> dnChain;
+ private final Map<String, String> attributes;
+
+ public AuthorizeDownloadAction(List<String> dnChain, Map<String, String> attributes) {
+ this.dnChain = dnChain;
+ this.attributes = attributes;
+ }
+
+ @Override
+ public DownloadAuthorization execute(DAOFactory daoFactory, AuthorityProvider authorityProvider) {
+ try {
+ return authorityProvider.authorizeDownload(dnChain, attributes);
+ } catch (UnknownIdentityException uie) {
+ throw new AccountNotFoundException(uie.getMessage(), uie);
+ } catch (AuthorityAccessException aae) {
+ throw new AdministrationException(aae.getMessage(), aae);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/AuthorizeUserAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/AuthorizeUserAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/AuthorizeUserAction.java
new file mode 100644
index 0000000..fe32772
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/AuthorizeUserAction.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import java.util.Calendar;
+import java.util.Date;
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.admin.dao.DataAccessException;
+import org.apache.nifi.admin.dao.UserDAO;
+import org.apache.nifi.admin.service.AccountDisabledException;
+import org.apache.nifi.admin.service.AccountNotFoundException;
+import org.apache.nifi.admin.service.AccountPendingException;
+import org.apache.nifi.admin.service.AdministrationException;
+import org.apache.nifi.authorization.AuthorityProvider;
+import org.apache.nifi.authorization.exception.AuthorityAccessException;
+import org.apache.nifi.authorization.exception.UnknownIdentityException;
+import org.apache.nifi.security.util.CertificateUtils;
+import org.apache.nifi.user.AccountStatus;
+import org.apache.nifi.user.NiFiUser;
+
+/**
+ *
+ */
+public class AuthorizeUserAction extends AbstractUserAction<NiFiUser> {
+
+ private final String dn;
+ private final int cacheDurationSeconds;
+
+ public AuthorizeUserAction(String dn, int cacheDurationSeconds) {
+ this.dn = dn;
+ this.cacheDurationSeconds = cacheDurationSeconds;
+ }
+
+ @Override
+ public NiFiUser execute(DAOFactory daoFactory, AuthorityProvider authorityProvider) throws DataAccessException {
+ UserDAO userDao = daoFactory.getUserDAO();
+
+ // get the user
+ NiFiUser user = userDao.findUserByDn(dn);
+
+ // verify the user was found
+ if (user == null) {
+ // determine whether this users exists
+ boolean doesDnExist = false;
+ try {
+ doesDnExist = authorityProvider.doesDnExist(dn);
+ } catch (AuthorityAccessException aae) {
+ throw new AdministrationException(String.format("Unable to access authority details: %s", aae.getMessage()), aae);
+ }
+
+ // if the authority provider has the details for this user, create the account
+ if (doesDnExist) {
+ // create the user
+ user = new NiFiUser();
+ user.setDn(dn);
+ user.setUserName(CertificateUtils.extractUsername(dn));
+ user.setJustification("User details specified by authority provider.");
+
+ try {
+ // verify the users account
+ verifyAccount(authorityProvider, user);
+
+ // get the date used for verification
+ Date now = user.getLastVerified();
+
+ // update the last accessed field
+ user.setLastAccessed(now);
+ user.setCreation(now);
+
+ // create the new user account
+ CreateUserAction createUser = new CreateUserAction(user);
+ createUser.execute(daoFactory, authorityProvider);
+ } catch (UnknownIdentityException uie) {
+ // strange since the provider just reported this dn existed but handleing anyways...
+ throw new AccountNotFoundException(String.format("Unable to verify access for %s.", dn));
+ } catch (AuthorityAccessException aae) {
+ throw new AdministrationException(String.format("Unable to access authority details: %s", aae.getMessage()), aae);
+ }
+ } else {
+ throw new AccountNotFoundException(String.format("Unable to verify access for %s.", dn));
+ }
+ } else {
+ Throwable providerError = null;
+
+ // verify the users account if necessary
+ if (isAccountVerificationRequired(user)) {
+ try {
+ // verify the users account
+ verifyAccount(authorityProvider, user);
+
+ // update the last accessed field
+ user.setLastAccessed(user.getLastVerified());
+ } catch (UnknownIdentityException uie) {
+ // check the account status before attempting to update the account - depending on the account
+ // status we might not need to update the account
+ checkAccountStatus(user);
+
+ // the user is currently active and they were not found in the providers - disable the account...
+ user.setStatus(AccountStatus.DISABLED);
+
+ // record the exception
+ providerError = uie;
+ } catch (AuthorityAccessException aae) {
+ throw new AdministrationException(String.format("Unable to access authority details: %s", aae.getMessage()), aae);
+ }
+ } else {
+ // verfiy the users account status before allowing access.
+ checkAccountStatus(user);
+
+ // update the users last accessed time
+ user.setLastAccessed(new Date());
+ }
+
+ // persist the user's updates
+ UpdateUserCacheAction updateUser = new UpdateUserCacheAction(user);
+ updateUser.execute(daoFactory, authorityProvider);
+
+ // persist the user's authorities
+ UpdateUserAuthoritiesCacheAction updateUserAuthorities = new UpdateUserAuthoritiesCacheAction(user);
+ updateUserAuthorities.execute(daoFactory, authorityProvider);
+
+ if (providerError != null) {
+ throw new AccountDisabledException(String.format("User credentials for %s were not found. This account has been disabled.", user.getDn()), providerError);
+ }
+ }
+
+ return user;
+ }
+
+ /**
+ * @return Determines if account verification is required
+ */
+ private boolean isAccountVerificationRequired(NiFiUser user) {
+ // accounts that have never been verified obviously needs to be re-verified
+ if (user.getLastVerified() == null) {
+ return true;
+ }
+
+ // create a calendar and substract the threshold - anything
+ // before this time will need to be re-verified
+ Calendar calendar = Calendar.getInstance();
+ calendar.add(Calendar.SECOND, -cacheDurationSeconds);
+
+ return user.getLastVerified().before(calendar.getTime());
+ }
+
+ /**
+ * Checks the account status of the specified user.
+ *
+ * @param user to check
+ */
+ private void checkAccountStatus(NiFiUser user) {
+ if (AccountStatus.DISABLED.equals(user.getStatus())) {
+ throw new AccountDisabledException(String.format("Account for %s is disabled.", user.getDn()));
+ } else if (AccountStatus.PENDING.equals(user.getStatus())) {
+ throw new AccountPendingException(String.format("Account for %s is pending.", user.getDn()));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/CreateUserAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/CreateUserAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/CreateUserAction.java
new file mode 100644
index 0000000..3833abb
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/CreateUserAction.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import java.util.Set;
+import org.apache.nifi.admin.dao.AuthorityDAO;
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.admin.dao.DataAccessException;
+import org.apache.nifi.admin.dao.UserDAO;
+import org.apache.nifi.authorization.Authority;
+import org.apache.nifi.authorization.AuthorityProvider;
+import org.apache.nifi.user.NiFiUser;
+
+/**
+ * Action for creating a NiFiUser account.
+ */
+public class CreateUserAction extends AbstractUserAction<Void> {
+
+ private final NiFiUser user;
+
+ public CreateUserAction(NiFiUser user) {
+ this.user = user;
+ }
+
+ @Override
+ public Void execute(DAOFactory daoFactory, AuthorityProvider authorityProvider) throws DataAccessException {
+ UserDAO userDao = daoFactory.getUserDAO();
+ AuthorityDAO authorityDao = daoFactory.getAuthorityDAO();
+
+ // create the user entry
+ userDao.createUser(user);
+
+ // create the authorities
+ Set<Authority> authorities = user.getAuthorities();
+ authorityDao.createAuthorities(authorities, user.getId());
+
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/DeleteUserAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/DeleteUserAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/DeleteUserAction.java
new file mode 100644
index 0000000..0d59b43
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/DeleteUserAction.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import org.apache.nifi.admin.dao.AuthorityDAO;
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.admin.dao.DataAccessException;
+import org.apache.nifi.admin.dao.UserDAO;
+import org.apache.nifi.admin.service.AccountNotFoundException;
+import org.apache.nifi.authorization.AuthorityProvider;
+import org.apache.nifi.user.AccountStatus;
+import org.apache.nifi.user.NiFiUser;
+
+/**
+ *
+ */
+public class DeleteUserAction implements AdministrationAction<Void> {
+
+ private final String userId;
+
+ /**
+ * Creates a new transactions for deleting the specified user.
+ *
+ * @param userId user identifier
+ */
+ public DeleteUserAction(String userId) {
+ this.userId = userId;
+ }
+
+ @Override
+ public Void execute(DAOFactory daoFactory, AuthorityProvider authorityProvider) throws DataAccessException {
+ final AuthorityDAO authorityDAO = daoFactory.getAuthorityDAO();
+ final UserDAO userDAO = daoFactory.getUserDAO();
+
+ // find the user and ensure they are currently revoked
+ final NiFiUser user = userDAO.findUserById(userId);
+
+ // ensure the user was found
+ if (user == null) {
+ throw new AccountNotFoundException(String.format("Unable to find account with ID %s.", userId));
+ }
+
+ // ensure the user is in the appropriate state
+ if (AccountStatus.ACTIVE.equals(user.getStatus())) {
+ throw new IllegalStateException(String.format("An active user cannot be removed. Revoke user access before attempting to remove."));
+ }
+
+ // remove the user and their authorities
+ authorityDAO.deleteAuthorities(userId);
+ userDAO.deleteUser(userId);
+
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/DisableUserAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/DisableUserAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/DisableUserAction.java
new file mode 100644
index 0000000..c31f107
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/DisableUserAction.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.admin.dao.DataAccessException;
+import org.apache.nifi.admin.dao.UserDAO;
+import org.apache.nifi.admin.service.AccountNotFoundException;
+import org.apache.nifi.admin.service.AdministrationException;
+import org.apache.nifi.authorization.AuthorityProvider;
+import org.apache.nifi.authorization.exception.AuthorityAccessException;
+import org.apache.nifi.authorization.exception.UnknownIdentityException;
+import org.apache.nifi.user.AccountStatus;
+import org.apache.nifi.user.NiFiUser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ */
+public class DisableUserAction implements AdministrationAction<NiFiUser> {
+
+ private static final Logger logger = LoggerFactory.getLogger(DisableUserAction.class);
+
+ private final String id;
+
+ public DisableUserAction(String id) {
+ this.id = id;
+ }
+
+ @Override
+ public NiFiUser execute(DAOFactory daoFactory, AuthorityProvider authorityProvider) throws DataAccessException {
+ UserDAO userDao = daoFactory.getUserDAO();
+
+ // get the user
+ NiFiUser user = userDao.findUserById(id);
+
+ // ensure the user exists
+ if (user == null) {
+ throw new AccountNotFoundException(String.format("Unable to find account with ID %s.", id));
+ }
+
+ // update the account
+ user.setStatus(AccountStatus.DISABLED);
+ user.setUserGroup(null);
+
+ // update the user locally
+ userDao.updateUser(user);
+
+ try {
+ // revoke the user in the authority provider
+ authorityProvider.revokeUser(user.getDn());
+ } catch (UnknownIdentityException uie) {
+ // user identity is not known
+ logger.info(String.format("User %s has already been removed from the authority provider.", user.getDn()));
+ } catch (AuthorityAccessException aae) {
+ throw new AdministrationException(String.format("Unable to revoke user '%s': %s", user.getDn(), aae.getMessage()), aae);
+ }
+
+ return user;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/DisableUserGroupAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/DisableUserGroupAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/DisableUserGroupAction.java
new file mode 100644
index 0000000..385fce6
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/DisableUserGroupAction.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.admin.dao.DataAccessException;
+import org.apache.nifi.admin.dao.UserDAO;
+import org.apache.nifi.admin.service.AdministrationException;
+import org.apache.nifi.authorization.AuthorityProvider;
+import org.apache.nifi.authorization.exception.AuthorityAccessException;
+import org.apache.nifi.authorization.exception.UnknownIdentityException;
+import org.apache.nifi.user.AccountStatus;
+import org.apache.nifi.user.NiFiUserGroup;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ */
+public class DisableUserGroupAction implements AdministrationAction<NiFiUserGroup> {
+
+ private static final Logger logger = LoggerFactory.getLogger(DisableUserGroupAction.class);
+
+ private final String group;
+
+ public DisableUserGroupAction(final String group) {
+ this.group = group;
+ }
+
+ @Override
+ public NiFiUserGroup execute(DAOFactory daoFactory, AuthorityProvider authorityProvider) throws DataAccessException {
+ final NiFiUserGroup userGroup = new NiFiUserGroup();
+
+ final UserDAO userDao = daoFactory.getUserDAO();
+
+ // update the user group locally
+ userDao.updateGroupStatus(group, AccountStatus.DISABLED);
+
+ // populate the group details
+ userGroup.setGroup(group);
+ userGroup.setUsers(userDao.findUsersForGroup(group));
+
+ try {
+ // revoke the user in the authority provider
+ authorityProvider.revokeGroup(group);
+ } catch (UnknownIdentityException uie) {
+ // user identity is not known
+ logger.info(String.format("User group %s has already been removed from the authority provider.", group));
+ } catch (AuthorityAccessException aae) {
+ throw new AdministrationException(String.format("Unable to revoke user group '%s': %s", group, aae.getMessage()), aae);
+ }
+
+ return userGroup;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/FindUserByDnAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/FindUserByDnAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/FindUserByDnAction.java
new file mode 100644
index 0000000..8e5b574
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/FindUserByDnAction.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.admin.dao.DataAccessException;
+import org.apache.nifi.admin.dao.UserDAO;
+import org.apache.nifi.authorization.AuthorityProvider;
+import org.apache.nifi.user.NiFiUser;
+
+/**
+ *
+ */
+public class FindUserByDnAction implements AdministrationAction<NiFiUser> {
+
+ private final String dn;
+
+ /**
+ * Creates a new transactions for getting a user with the specified DN.
+ *
+ * @param dn The DN of the user to obtain
+ */
+ public FindUserByDnAction(String dn) {
+ this.dn = dn;
+ }
+
+ @Override
+ public NiFiUser execute(DAOFactory daoFactory, AuthorityProvider authorityProvider) throws DataAccessException {
+ // get a UserDAO
+ UserDAO userDAO = daoFactory.getUserDAO();
+
+ // return the desired user
+ return userDAO.findUserByDn(dn);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/FindUserByIdAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/FindUserByIdAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/FindUserByIdAction.java
new file mode 100644
index 0000000..0a10841
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/FindUserByIdAction.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.admin.dao.DataAccessException;
+import org.apache.nifi.admin.dao.UserDAO;
+import org.apache.nifi.authorization.AuthorityProvider;
+import org.apache.nifi.user.NiFiUser;
+
+public class FindUserByIdAction implements AdministrationAction<NiFiUser> {
+
+ private final String id;
+
+ /**
+ * Creates a new transactions for getting a user with the specified id.
+ *
+ * @param id of user
+ */
+ public FindUserByIdAction(String id) {
+ this.id = id;
+ }
+
+ @Override
+ public NiFiUser execute(DAOFactory daoFactory, AuthorityProvider authorityProvider) throws DataAccessException {
+ // get a UserDAO
+ UserDAO userDAO = daoFactory.getUserDAO();
+
+ // return the desired user
+ return userDAO.findUserById(id);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/GetActionAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/GetActionAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/GetActionAction.java
new file mode 100644
index 0000000..1dc5588
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/GetActionAction.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import org.apache.nifi.action.Action;
+import org.apache.nifi.admin.dao.ActionDAO;
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.authorization.AuthorityProvider;
+
+/**
+ * Gets the action with the specified id.
+ */
+public class GetActionAction implements AdministrationAction<Action> {
+
+ private final Integer id;
+
+ public GetActionAction(Integer id) {
+ this.id = id;
+ }
+
+ @Override
+ public Action execute(DAOFactory daoFactory, AuthorityProvider authorityProvider) {
+ ActionDAO actionDao = daoFactory.getActionDAO();
+ return actionDao.getAction(id);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/GetActionsAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/GetActionsAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/GetActionsAction.java
new file mode 100644
index 0000000..3b82d79
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/GetActionsAction.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import java.util.Date;
+import org.apache.nifi.admin.dao.ActionDAO;
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.authorization.AuthorityProvider;
+import org.apache.nifi.history.History;
+import org.apache.nifi.history.HistoryQuery;
+
+/**
+ * Get all actions that match the specified query.
+ */
+public class GetActionsAction implements AdministrationAction<History> {
+
+ private final HistoryQuery query;
+
+ public GetActionsAction(HistoryQuery query) {
+ this.query = query;
+ }
+
+ @Override
+ public History execute(DAOFactory daoFactory, AuthorityProvider authorityProvider) {
+ ActionDAO actionDao = daoFactory.getActionDAO();
+
+ // find all matching history
+ History history = actionDao.findActions(query);
+ history.setLastRefreshed(new Date());
+
+ return history;
+ }
+
+}
[39/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardProvenanceEventRecord.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardProvenanceEventRecord.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardProvenanceEventRecord.java
new file mode 100644
index 0000000..4eb7001
--- /dev/null
+++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardProvenanceEventRecord.java
@@ -0,0 +1,751 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.processor.Relationship;
+
+/**
+ * Holder for provenance relevant information
+ * <p/>
+ */
+public final class StandardProvenanceEventRecord implements ProvenanceEventRecord {
+
+ private final long eventTime;
+ private final long entryDate;
+ private final ProvenanceEventType eventType;
+ private final long lineageStartDate;
+ private final Set<String> lineageIdentifiers;
+ private final String componentId;
+ private final String componentType;
+ private final String transitUri;
+ private final String sourceSystemFlowFileIdentifier;
+ private final String uuid;
+ private final List<String> parentUuids;
+ private final List<String> childrenUuids;
+ private final String alternateIdentifierUri;
+ private final String details;
+ private final String relationship;
+ private final long storageByteOffset;
+ private final String storageFilename;
+ private final long eventDuration;
+
+ private final String contentClaimSection;
+ private final String contentClaimContainer;
+ private final String contentClaimIdentifier;
+ private final Long contentClaimOffset;
+ private final long contentSize;
+
+ private final String previousClaimSection;
+ private final String previousClaimContainer;
+ private final String previousClaimIdentifier;
+ private final Long previousClaimOffset;
+ private final Long previousSize;
+
+ private final String sourceQueueIdentifier;
+
+ private final Map<String, String> previousAttributes;
+ private final Map<String, String> updatedAttributes;
+
+ private volatile long eventId;
+
+ private StandardProvenanceEventRecord(final Builder builder) {
+ this.eventTime = builder.eventTime;
+ this.entryDate = builder.entryDate;
+ this.eventType = builder.eventType;
+ this.componentId = builder.componentId;
+ this.componentType = builder.componentType;
+ this.transitUri = builder.transitUri;
+ this.sourceSystemFlowFileIdentifier = builder.sourceSystemFlowFileIdentifier;
+ this.uuid = builder.uuid;
+ this.parentUuids = builder.parentUuids;
+ this.childrenUuids = builder.childrenUuids;
+ this.alternateIdentifierUri = builder.alternateIdentifierUri;
+ this.details = builder.details;
+ this.relationship = builder.relationship;
+ this.storageByteOffset = builder.storageByteOffset;
+ this.storageFilename = builder.storageFilename;
+ this.eventDuration = builder.eventDuration;
+ this.lineageStartDate = builder.lineageStartDate;
+ this.lineageIdentifiers = Collections.unmodifiableSet(builder.lineageIdentifiers);
+
+ previousClaimSection = builder.previousClaimSection;
+ previousClaimContainer = builder.previousClaimContainer;
+ previousClaimIdentifier = builder.previousClaimIdentifier;
+ previousClaimOffset = builder.previousClaimOffset;
+ previousSize = builder.previousSize;
+
+ contentClaimSection = builder.contentClaimSection;
+ contentClaimContainer = builder.contentClaimContainer;
+ contentClaimIdentifier = builder.contentClaimIdentifier;
+ contentClaimOffset = builder.contentClaimOffset;
+ contentSize = builder.contentSize;
+
+ previousAttributes = builder.previousAttributes == null ? Collections.<String, String>emptyMap() : Collections.unmodifiableMap(builder.previousAttributes);
+ updatedAttributes = builder.updatedAttributes == null ? Collections.<String, String>emptyMap() : Collections.unmodifiableMap(builder.updatedAttributes);
+
+ sourceQueueIdentifier = builder.sourceQueueIdentifier;
+
+ }
+
+ public String getStorageFilename() {
+ return storageFilename;
+ }
+
+ public long getStorageByteOffset() {
+ return storageByteOffset;
+ }
+
+ void setEventId(final long eventId) {
+ this.eventId = eventId;
+ }
+
+ @Override
+ public long getEventId() {
+ return eventId;
+ }
+
+ @Override
+ public long getEventTime() {
+ return eventTime;
+ }
+
+ @Override
+ public Set<String> getLineageIdentifiers() {
+ return lineageIdentifiers;
+ }
+
+ @Override
+ public long getLineageStartDate() {
+ return lineageStartDate;
+ }
+
+ @Override
+ public long getFileSize() {
+ return contentSize;
+ }
+
+ @Override
+ public Long getPreviousFileSize() {
+ return previousSize;
+ }
+
+ @Override
+ public ProvenanceEventType getEventType() {
+ return eventType;
+ }
+
+ @Override
+ public Map<String, String> getAttributes() {
+ final Map<String, String> allAttrs = new HashMap<>(previousAttributes.size() + updatedAttributes.size());
+ allAttrs.putAll(previousAttributes);
+ for (final Map.Entry<String, String> entry : updatedAttributes.entrySet()) {
+ if (entry.getValue() != null) {
+ allAttrs.put(entry.getKey(), entry.getValue());
+ }
+ }
+ return allAttrs;
+ }
+
+ @Override
+ public String getComponentId() {
+ return componentId;
+ }
+
+ @Override
+ public String getComponentType() {
+ return componentType;
+ }
+
+ @Override
+ public String getTransitUri() {
+ return transitUri;
+ }
+
+ @Override
+ public String getSourceSystemFlowFileIdentifier() {
+ return sourceSystemFlowFileIdentifier;
+ }
+
+ @Override
+ public String getFlowFileUuid() {
+ return uuid;
+ }
+
+ @Override
+ public List<String> getParentUuids() {
+ return parentUuids == null ? Collections.<String>emptyList() : parentUuids;
+ }
+
+ @Override
+ public List<String> getChildUuids() {
+ return childrenUuids == null ? Collections.<String>emptyList() : childrenUuids;
+ }
+
+ @Override
+ public String getAlternateIdentifierUri() {
+ return alternateIdentifierUri;
+ }
+
+ @Override
+ public long getEventDuration() {
+ return eventDuration;
+ }
+
+ @Override
+ public String getDetails() {
+ return details;
+ }
+
+ @Override
+ public String getRelationship() {
+ return relationship;
+ }
+
+ @Override
+ public long getFlowFileEntryDate() {
+ return entryDate;
+ }
+
+ @Override
+ public String getContentClaimSection() {
+ return contentClaimSection;
+ }
+
+ @Override
+ public String getContentClaimContainer() {
+ return contentClaimContainer;
+ }
+
+ @Override
+ public String getContentClaimIdentifier() {
+ return contentClaimIdentifier;
+ }
+
+ @Override
+ public Long getContentClaimOffset() {
+ return contentClaimOffset;
+ }
+
+ @Override
+ public String getSourceQueueIdentifier() {
+ return sourceQueueIdentifier;
+ }
+
+ @Override
+ public Map<String, String> getPreviousAttributes() {
+ return previousAttributes;
+ }
+
+ @Override
+ public String getPreviousContentClaimContainer() {
+ return previousClaimContainer;
+ }
+
+ @Override
+ public String getPreviousContentClaimIdentifier() {
+ return previousClaimIdentifier;
+ }
+
+ @Override
+ public Long getPreviousContentClaimOffset() {
+ return previousClaimOffset;
+ }
+
+ @Override
+ public String getPreviousContentClaimSection() {
+ return previousClaimSection;
+ }
+
+ @Override
+ public Map<String, String> getUpdatedAttributes() {
+ return updatedAttributes;
+ }
+
+ @Override
+ public int hashCode() {
+ final int eventTypeCode;
+ if (eventType == ProvenanceEventType.CLONE || eventType == ProvenanceEventType.JOIN || eventType == ProvenanceEventType.FORK) {
+ eventTypeCode = 1472;
+ } else if (eventType == ProvenanceEventType.REPLAY) {
+ eventTypeCode = 21479 + (int) (0x7FFFFFFF & eventTime); // use lower bits of event time.
+ } else {
+ eventTypeCode = 4812 + eventType.hashCode() + 4 * uuid.hashCode();
+ }
+
+ return -37423 + 3 * componentId.hashCode() + (transitUri == null ? 0 : 41 * transitUri.hashCode())
+ + (relationship == null ? 0 : 47 * relationship.hashCode()) + 44 * eventTypeCode;
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (obj == null) {
+ return false;
+ }
+ if (obj == this) {
+ return true;
+ }
+ if (!(obj instanceof StandardProvenanceEventRecord)) {
+ return false;
+ }
+
+ final StandardProvenanceEventRecord other = (StandardProvenanceEventRecord) obj;
+ // If event ID's are populated and not equal, return false. If they have not yet been populated, do not
+ // use them in the comparison.
+ if (eventId > 0L && other.getEventId() > 0L && eventId != other.getEventId()) {
+ return false;
+ }
+ if (eventType != other.eventType) {
+ return false;
+ }
+
+ if (!componentId.equals(other.componentId)) {
+ return false;
+ }
+
+ if (different(parentUuids, other.parentUuids)) {
+ return false;
+ }
+
+ if (different(childrenUuids, other.childrenUuids)) {
+ return false;
+ }
+
+ // SPAWN had issues indicating which should be the event's FlowFileUUID in the case that there is 1 parent and 1 child.
+ if (!uuid.equals(other.uuid)) {
+ return false;
+ }
+
+ if (different(transitUri, other.transitUri)) {
+ return false;
+ }
+
+ if (different(relationship, other.relationship)) {
+ return false;
+ }
+
+ return !(eventType == ProvenanceEventType.REPLAY && eventTime != other.getEventTime());
+ }
+
+ private boolean different(final Object a, final Object b) {
+ if (a == null && b == null) {
+ return false;
+ }
+ if (a == null || b == null) {
+ return true;
+ }
+
+ return !a.equals(b);
+ }
+
+ private boolean different(final List<String> a, final List<String> b) {
+ if (a == null && b == null) {
+ return false;
+ }
+
+ if (a == null && b != null) {
+ return true;
+ }
+
+ if (a != null && b == null) {
+ return true;
+ }
+
+ if (a.size() != b.size()) {
+ return true;
+ }
+
+ final List<String> sortedA = new ArrayList<>(a);
+ final List<String> sortedB = new ArrayList<>(b);
+
+ Collections.sort(sortedA);
+ Collections.sort(sortedB);
+
+ for (int i = 0; i < sortedA.size(); i++) {
+ if (!sortedA.get(i).equals(sortedB.get(i))) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return "ProvenanceEventRecord ["
+ + "eventId=" + eventId
+ + ", eventType=" + eventType
+ + ", eventTime=" + new Date(eventTime)
+ + ", uuid=" + uuid
+ + ", fileSize=" + contentSize
+ + ", componentId=" + componentId
+ + ", transitUri=" + transitUri
+ + ", sourceSystemFlowFileIdentifier=" + sourceSystemFlowFileIdentifier
+ + ", parentUuids=" + parentUuids
+ + ", alternateIdentifierUri=" + alternateIdentifierUri + "]";
+ }
+
+ public static class Builder implements ProvenanceEventBuilder {
+
+ private long eventTime = System.currentTimeMillis();
+ private long entryDate;
+ private long lineageStartDate;
+ private Set<String> lineageIdentifiers = new HashSet<>();
+ private ProvenanceEventType eventType = null;
+ private String componentId = null;
+ private String componentType = null;
+ private String sourceSystemFlowFileIdentifier = null;
+ private String transitUri = null;
+ private String uuid = null;
+ private List<String> parentUuids = null;
+ private List<String> childrenUuids = null;
+ private String contentType = null;
+ private String alternateIdentifierUri = null;
+ private String details = null;
+ private String relationship = null;
+ private long storageByteOffset = -1L;
+ private long eventDuration = -1L;
+ private String storageFilename;
+
+ private String contentClaimSection;
+ private String contentClaimContainer;
+ private String contentClaimIdentifier;
+ private Long contentClaimOffset;
+ private Long contentSize;
+
+ private String previousClaimSection;
+ private String previousClaimContainer;
+ private String previousClaimIdentifier;
+ private Long previousClaimOffset;
+ private Long previousSize;
+
+ private String sourceQueueIdentifier;
+
+ private Map<String, String> previousAttributes;
+ private Map<String, String> updatedAttributes;
+
+ @Override
+ public Builder fromEvent(final ProvenanceEventRecord event) {
+ eventTime = event.getEventTime();
+ entryDate = event.getFlowFileEntryDate();
+ lineageStartDate = event.getLineageStartDate();
+ lineageIdentifiers = event.getLineageIdentifiers();
+ eventType = event.getEventType();
+ componentId = event.getComponentId();
+ componentType = event.getComponentType();
+ transitUri = event.getTransitUri();
+ sourceSystemFlowFileIdentifier = event.getSourceSystemFlowFileIdentifier();
+ uuid = event.getFlowFileUuid();
+ parentUuids = event.getParentUuids();
+ childrenUuids = event.getChildUuids();
+ alternateIdentifierUri = event.getAlternateIdentifierUri();
+ eventDuration = event.getEventDuration();
+ previousAttributes = event.getPreviousAttributes();
+ updatedAttributes = event.getUpdatedAttributes();
+ details = event.getDetails();
+ relationship = event.getRelationship();
+
+ contentClaimSection = event.getContentClaimSection();
+ contentClaimContainer = event.getContentClaimContainer();
+ contentClaimIdentifier = event.getContentClaimIdentifier();
+ contentClaimOffset = event.getContentClaimOffset();
+ contentSize = event.getFileSize();
+
+ previousClaimSection = event.getPreviousContentClaimSection();
+ previousClaimContainer = event.getPreviousContentClaimContainer();
+ previousClaimIdentifier = event.getPreviousContentClaimIdentifier();
+ previousClaimOffset = event.getPreviousContentClaimOffset();
+ previousSize = event.getPreviousFileSize();
+
+ sourceQueueIdentifier = event.getSourceQueueIdentifier();
+
+ if (event instanceof StandardProvenanceEventRecord) {
+ final StandardProvenanceEventRecord standardProvEvent = (StandardProvenanceEventRecord) event;
+ storageByteOffset = standardProvEvent.storageByteOffset;
+ storageFilename = standardProvEvent.storageFilename;
+ }
+
+ return this;
+ }
+
+ @Override
+ public Builder setFlowFileEntryDate(final long entryDate) {
+ this.entryDate = entryDate;
+ return this;
+ }
+
+ @Override
+ public Builder setLineageIdentifiers(final Set<String> lineageIdentifiers) {
+ this.lineageIdentifiers = lineageIdentifiers;
+ return this;
+ }
+
+ @Override
+ public Builder setAttributes(final Map<String, String> previousAttributes, final Map<String, String> updatedAttributes) {
+ this.previousAttributes = previousAttributes;
+ this.updatedAttributes = updatedAttributes;
+ return this;
+ }
+
+ @Override
+ public Builder setFlowFileUUID(final String uuid) {
+ this.uuid = uuid;
+ return this;
+ }
+
+ public Builder setStorageLocation(final String filename, final long offset) {
+ this.storageFilename = filename;
+ this.storageByteOffset = offset;
+ return this;
+ }
+
+ @Override
+ public Builder setEventTime(long eventTime) {
+ this.eventTime = eventTime;
+ return this;
+ }
+
+ @Override
+ public Builder setEventDuration(final long millis) {
+ this.eventDuration = millis;
+ return this;
+ }
+
+ @Override
+ public Builder setLineageStartDate(final long startDate) {
+ this.lineageStartDate = startDate;
+ return this;
+ }
+
+ public Builder addLineageIdentifier(final String lineageIdentifier) {
+ this.lineageIdentifiers.add(lineageIdentifier);
+ return this;
+ }
+
+ @Override
+ public Builder setEventType(ProvenanceEventType eventType) {
+ this.eventType = eventType;
+ return this;
+ }
+
+ @Override
+ public Builder setComponentId(String componentId) {
+ this.componentId = componentId;
+ return this;
+ }
+
+ @Override
+ public Builder setComponentType(String componentType) {
+ this.componentType = componentType;
+ return this;
+ }
+
+ @Override
+ public Builder setSourceSystemFlowFileIdentifier(String sourceSystemFlowFileIdentifier) {
+ this.sourceSystemFlowFileIdentifier = sourceSystemFlowFileIdentifier;
+ return this;
+ }
+
+ @Override
+ public Builder setTransitUri(String transitUri) {
+ this.transitUri = transitUri;
+ return this;
+ }
+
+ @Override
+ public Builder addParentFlowFile(final FlowFile parentFlowFile) {
+ if (this.parentUuids == null) {
+ this.parentUuids = new ArrayList<>();
+ }
+ this.parentUuids.add(parentFlowFile.getAttribute(CoreAttributes.UUID.key()));
+ return this;
+ }
+
+ public Builder addParentUuid(final String uuid) {
+ if (this.parentUuids == null) {
+ this.parentUuids = new ArrayList<>();
+ }
+ this.parentUuids.add(uuid);
+ return this;
+ }
+
+ @Override
+ public Builder removeParentFlowFile(final FlowFile parentFlowFile) {
+ if (this.parentUuids == null) {
+ return this;
+ }
+
+ parentUuids.remove(parentFlowFile.getAttribute(CoreAttributes.UUID.key()));
+ return this;
+ }
+
+ @Override
+ public Builder addChildFlowFile(final FlowFile childFlowFile) {
+ if (this.childrenUuids == null) {
+ this.childrenUuids = new ArrayList<>();
+ }
+ this.childrenUuids.add(childFlowFile.getAttribute(CoreAttributes.UUID.key()));
+ return this;
+ }
+
+ public Builder addChildUuid(final String uuid) {
+ if (this.childrenUuids == null) {
+ this.childrenUuids = new ArrayList<>();
+ }
+ this.childrenUuids.add(uuid);
+ return this;
+ }
+
+ @Override
+ public Builder removeChildFlowFile(final FlowFile childFlowFile) {
+ if (this.childrenUuids == null) {
+ return this;
+ }
+
+ childrenUuids.remove(childFlowFile.getAttribute(CoreAttributes.UUID.key()));
+ return this;
+ }
+
+ public Builder setContentType(String contentType) {
+ this.contentType = contentType;
+ return this;
+ }
+
+ @Override
+ public Builder setAlternateIdentifierUri(String alternateIdentifierUri) {
+ this.alternateIdentifierUri = alternateIdentifierUri;
+ return this;
+ }
+
+ @Override
+ public Builder setDetails(String details) {
+ this.details = details;
+ return this;
+ }
+
+ @Override
+ public Builder setRelationship(Relationship relationship) {
+ this.relationship = relationship.getName();
+ return this;
+ }
+
+ public Builder setRelationship(final String relationship) {
+ this.relationship = relationship;
+ return this;
+ }
+
+ @Override
+ public ProvenanceEventBuilder fromFlowFile(final FlowFile flowFile) {
+ setFlowFileEntryDate(flowFile.getEntryDate());
+ setLineageIdentifiers(flowFile.getLineageIdentifiers());
+ setLineageStartDate(flowFile.getLineageStartDate());
+ setAttributes(Collections.<String, String>emptyMap(), flowFile.getAttributes());
+ uuid = flowFile.getAttribute(CoreAttributes.UUID.key());
+ this.contentSize = flowFile.getSize();
+ return this;
+ }
+
+ @Override
+ public Builder setPreviousContentClaim(final String container, final String section, final String identifier, final Long offset, final long size) {
+ previousClaimSection = section;
+ previousClaimContainer = container;
+ previousClaimIdentifier = identifier;
+ previousClaimOffset = offset;
+ previousSize = size;
+ return this;
+ }
+
+ @Override
+ public Builder setCurrentContentClaim(final String container, final String section, final String identifier, final Long offset, final long size) {
+ contentClaimSection = section;
+ contentClaimContainer = container;
+ contentClaimIdentifier = identifier;
+ contentClaimOffset = offset;
+ contentSize = size;
+ return this;
+ }
+
+ @Override
+ public Builder setSourceQueueIdentifier(final String identifier) {
+ sourceQueueIdentifier = identifier;
+ return this;
+ }
+
+ private void assertSet(final Object value, final String name) {
+ if (value == null) {
+ throw new IllegalStateException("Cannot create Provenance Event Record because " + name + " is not set");
+ }
+ }
+
+ public ProvenanceEventType getEventType() {
+ return eventType;
+ }
+
+ public List<String> getChildUuids() {
+ return Collections.unmodifiableList(childrenUuids);
+ }
+
+ public List<String> getParentUuids() {
+ return Collections.unmodifiableList(parentUuids);
+ }
+
+ @Override
+ public StandardProvenanceEventRecord build() {
+ assertSet(eventType, "Event Type");
+ assertSet(componentId, "Component ID");
+ assertSet(componentType, "Component Type");
+ assertSet(uuid, "FlowFile UUID");
+ assertSet(contentSize, "FlowFile Size");
+
+ switch (eventType) {
+ case ADDINFO:
+ if (alternateIdentifierUri == null) {
+ throw new IllegalStateException("Cannot create Provenance Event Record of type " + eventType + " because no alternate identifiers have been set");
+ }
+ break;
+ case RECEIVE:
+ case SEND:
+ assertSet(transitUri, "Transit URI");
+ break;
+ case ROUTE:
+ assertSet(relationship, "Relationship");
+ break;
+ case CLONE:
+ case FORK:
+ case JOIN:
+ if ((parentUuids == null || parentUuids.isEmpty()) && (childrenUuids == null || childrenUuids.isEmpty())) {
+ throw new IllegalStateException("Cannot create Provenance Event Record of type " + eventType + " because no Parent UUIDs or Children UUIDs have been set");
+ }
+ break;
+ default:
+ break;
+ }
+
+ return new StandardProvenanceEventRecord(this);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardQueryResult.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardQueryResult.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardQueryResult.java
new file mode 100644
index 0000000..9a9a27d
--- /dev/null
+++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardQueryResult.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.nifi.provenance.search.Query;
+import org.apache.nifi.provenance.search.QueryResult;
+
+public class StandardQueryResult implements QueryResult {
+
+ public static final int TTL = (int) TimeUnit.MILLISECONDS.convert(30, TimeUnit.MINUTES);
+ private final Query query;
+ private final long creationNanos;
+
+ private final int numSteps;
+ private final ReadWriteLock rwLock = new ReentrantReadWriteLock();
+ private final Lock readLock = rwLock.readLock();
+
+ private final Lock writeLock = rwLock.writeLock();
+ // guarded by writeLock
+ private final List<ProvenanceEventRecord> matchingRecords = new ArrayList<>();
+ private long totalHitCount;
+ private int numCompletedSteps = 0;
+ private Date expirationDate;
+ private String error;
+ private long queryTime;
+
+ private volatile boolean canceled = false;
+
+ public StandardQueryResult(final Query query, final int numSteps) {
+ this.query = query;
+ this.numSteps = numSteps;
+ this.creationNanos = System.nanoTime();
+
+ updateExpiration();
+ }
+
+ @Override
+ public List<ProvenanceEventRecord> getMatchingEvents() {
+ readLock.lock();
+ try {
+ if (matchingRecords.size() <= query.getMaxResults()) {
+ return new ArrayList<>(matchingRecords);
+ }
+
+ final List<ProvenanceEventRecord> copy = new ArrayList<>(query.getMaxResults());
+ for (int i = 0; i < query.getMaxResults(); i++) {
+ copy.add(matchingRecords.get(i));
+ }
+
+ return copy;
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public long getTotalHitCount() {
+ readLock.lock();
+ try {
+ return totalHitCount;
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public long getQueryTime() {
+ return queryTime;
+ }
+
+ @Override
+ public Date getExpiration() {
+ return expirationDate;
+ }
+
+ @Override
+ public String getError() {
+ return error;
+ }
+
+ @Override
+ public int getPercentComplete() {
+ readLock.lock();
+ try {
+ return (numSteps < 1) ? 100 : (int) (((float) numCompletedSteps / (float) numSteps) * 100.0F);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public boolean isFinished() {
+ readLock.lock();
+ try {
+ return numCompletedSteps >= numSteps || canceled;
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ void cancel() {
+ this.canceled = true;
+ }
+
+ public void setError(final String error) {
+ writeLock.lock();
+ try {
+ this.error = error;
+ numCompletedSteps++;
+
+ updateExpiration();
+ if (numCompletedSteps >= numSteps) {
+ final long searchNanos = System.nanoTime() - creationNanos;
+ queryTime = TimeUnit.MILLISECONDS.convert(searchNanos, TimeUnit.NANOSECONDS);
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ public void update(final Collection<ProvenanceEventRecord> matchingRecords, final long totalHits) {
+ writeLock.lock();
+ try {
+ this.matchingRecords.addAll(matchingRecords);
+ this.totalHitCount += totalHits;
+
+ numCompletedSteps++;
+ updateExpiration();
+
+ if (numCompletedSteps >= numSteps) {
+ final long searchNanos = System.nanoTime() - creationNanos;
+ queryTime = TimeUnit.MILLISECONDS.convert(searchNanos, TimeUnit.NANOSECONDS);
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ /**
+ * Must be called with write lock!
+ */
+ private void updateExpiration() {
+ expirationDate = new Date(System.currentTimeMillis() + TTL);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/lineage/EdgeNode.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/lineage/EdgeNode.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/lineage/EdgeNode.java
new file mode 100644
index 0000000..0aaf5ef
--- /dev/null
+++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/lineage/EdgeNode.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance.lineage;
+
+import static java.util.Objects.requireNonNull;
+
+public class EdgeNode implements LineageEdge {
+
+ private final String uuid;
+ private final LineageNode source;
+ private final LineageNode destination;
+
+ public EdgeNode(final String uuid, final LineageNode source, final LineageNode destination) {
+ this.uuid = uuid;
+ this.source = requireNonNull(source);
+ this.destination = requireNonNull(destination);
+ }
+
+ @Override
+ public String getUuid() {
+ return uuid;
+ }
+
+ @Override
+ public LineageNode getSource() {
+ return source;
+ }
+
+ @Override
+ public LineageNode getDestination() {
+ return destination;
+ }
+
+ @Override
+ public int hashCode() {
+ return 43298293 + source.hashCode() + destination.hashCode();
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+
+ if (!(obj instanceof EdgeNode)) {
+ return false;
+ }
+
+ final EdgeNode other = (EdgeNode) obj;
+ return (source.equals(other.source) && destination.equals(other.destination));
+ }
+
+ @Override
+ public String toString() {
+ return "Edge[Source=" + source + ", Destination=" + destination + "]";
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/lineage/EventNode.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/lineage/EventNode.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/lineage/EventNode.java
new file mode 100644
index 0000000..1f8d1dc
--- /dev/null
+++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/lineage/EventNode.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance.lineage;
+
+import java.util.List;
+
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.provenance.ProvenanceEventRecord;
+import org.apache.nifi.provenance.ProvenanceEventType;
+
+public class EventNode implements ProvenanceEventLineageNode {
+
+ private final ProvenanceEventRecord record;
+ private String clusterNodeIdentifier = null;
+
+ public EventNode(final ProvenanceEventRecord event) {
+ this.record = event;
+ }
+
+ @Override
+ public String getIdentifier() {
+ return String.valueOf(getEventIdentifier());
+ }
+
+ @Deprecated
+ @Override
+ public String getClusterNodeIdentifier() {
+ return clusterNodeIdentifier;
+ }
+
+ @Deprecated
+ public void setClusterNodeIdentifier(final String nodeIdentifier) {
+ this.clusterNodeIdentifier = nodeIdentifier;
+ }
+
+ @Override
+ public LineageNodeType getNodeType() {
+ return LineageNodeType.PROVENANCE_EVENT_NODE;
+ }
+
+ @Override
+ public ProvenanceEventType getEventType() {
+ return record.getEventType();
+ }
+
+ @Override
+ public long getTimestamp() {
+ return record.getEventTime();
+ }
+
+ @Override
+ public long getEventIdentifier() {
+ return record.getEventId();
+ }
+
+ @Override
+ public String getFlowFileUuid() {
+ return record.getAttributes().get(CoreAttributes.UUID.key());
+ }
+
+ @Override
+ public List<String> getParentUuids() {
+ return record.getParentUuids();
+ }
+
+ @Override
+ public List<String> getChildUuids() {
+ return record.getChildUuids();
+ }
+
+ @Override
+ public int hashCode() {
+ return 2938472 + record.hashCode();
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (obj == null) {
+ return false;
+ }
+ if (this == obj) {
+ return true;
+ }
+
+ if (!(obj instanceof EventNode)) {
+ return false;
+ }
+
+ final EventNode other = (EventNode) obj;
+ return record.equals(other.record);
+ }
+
+ @Override
+ public String toString() {
+ return "Event[ID=" + record.getEventId() + ", Type=" + record.getEventType() + ", UUID=" + record.getFlowFileUuid() + ", Component=" + record.getComponentId() + "]";
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/lineage/FlowFileLineage.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/lineage/FlowFileLineage.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/lineage/FlowFileLineage.java
new file mode 100644
index 0000000..c36c38d
--- /dev/null
+++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/lineage/FlowFileLineage.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance.lineage;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import static java.util.Objects.requireNonNull;
+
+public class FlowFileLineage implements Lineage {
+
+ private final List<LineageNode> nodes;
+ private final List<LineageEdge> edges;
+
+ public FlowFileLineage(final Collection<LineageNode> nodes, final Collection<LineageEdge> edges) {
+ this.nodes = new ArrayList<>(requireNonNull(nodes));
+ this.edges = new ArrayList<>(requireNonNull(edges));
+ }
+
+ @Override
+ public List<LineageNode> getNodes() {
+ return nodes;
+ }
+
+ @Override
+ public List<LineageEdge> getEdges() {
+ return edges;
+ }
+
+ @Override
+ public int hashCode() {
+ int sum = 923;
+ for (final LineageNode node : nodes) {
+ sum += node.hashCode();
+ }
+
+ for (final LineageEdge edge : edges) {
+ sum += edge.hashCode();
+ }
+
+ return sum;
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (obj == null) {
+ return false;
+ }
+
+ if (obj == this) {
+ return true;
+ }
+
+ if (!(obj instanceof FlowFileLineage)) {
+ return false;
+ }
+
+ final FlowFileLineage other = (FlowFileLineage) obj;
+ return nodes.equals(other.nodes) && edges.equals(other.edges);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/lineage/FlowFileNode.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/lineage/FlowFileNode.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/lineage/FlowFileNode.java
new file mode 100644
index 0000000..94e7661
--- /dev/null
+++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/lineage/FlowFileNode.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance.lineage;
+
+import static java.util.Objects.requireNonNull;
+
+public class FlowFileNode implements LineageNode {
+
+ private final String flowFileUuid;
+ private final long creationTime;
+ private String clusterNodeIdentifier;
+
+ public FlowFileNode(final String flowFileUuid, final long flowFileCreationTime) {
+ this.flowFileUuid = requireNonNull(flowFileUuid);
+ this.creationTime = flowFileCreationTime;
+ }
+
+ @Override
+ public String getIdentifier() {
+ return flowFileUuid;
+ }
+
+ @Override
+ public long getTimestamp() {
+ return creationTime;
+ }
+
+ @Deprecated
+ @Override
+ public String getClusterNodeIdentifier() {
+ return clusterNodeIdentifier;
+ }
+
+ @Override
+ public LineageNodeType getNodeType() {
+ return LineageNodeType.FLOWFILE_NODE;
+ }
+
+ @Override
+ public String getFlowFileUuid() {
+ return flowFileUuid;
+ }
+
+ @Override
+ public int hashCode() {
+ return 23498723 + flowFileUuid.hashCode();
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (obj == null) {
+ return false;
+ }
+ if (obj == this) {
+ return true;
+ }
+
+ if (!(obj instanceof FlowFileNode)) {
+ return false;
+ }
+
+ final FlowFileNode other = (FlowFileNode) obj;
+ return flowFileUuid.equals(other.flowFileUuid);
+ }
+
+ @Override
+ public String toString() {
+ return "FlowFile[UUID=" + flowFileUuid + "]";
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/pom.xml b/nifi-commons/nifi-expression-language/pom.xml
new file mode 100644
index 0000000..3f0b8be
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/pom.xml
@@ -0,0 +1,56 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-commons</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>nifi-expression-language</artifactId>
+ <packaging>jar</packaging>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.antlr</groupId>
+ <artifactId>antlr3-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>antlr</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <configuration>
+ <excludes>**/antlr/AttributeExpressionParser.java,**/antlr/AttributeExpressionLexer.java</excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>org.antlr</groupId>
+ <artifactId>antlr-runtime</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionLexer.g
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionLexer.g b/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionLexer.g
new file mode 100644
index 0000000..80581f5
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionLexer.g
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+lexer grammar AttributeExpressionLexer;
+
+@header {
+ package org.apache.nifi.attribute.expression.language.antlr;
+ import org.apache.nifi.attribute.expression.language.exception.AttributeExpressionLanguageParsingException;
+}
+
+@rulecatch {
+ catch(final Exception e) {
+ throw new AttributeExpressionLanguageParsingException(e);
+ }
+}
+
+@members {
+ public void displayRecognitionError(String[] tokenNames, RecognitionException e) {
+ final StringBuilder sb = new StringBuilder();
+ if ( e.token == null ) {
+ sb.append("Unrecognized token ");
+ } else {
+ sb.append("Unexpected token '").append(e.token.getText()).append("' ");
+ }
+ sb.append("at line ").append(e.line);
+ if ( e.approximateLineInfo ) {
+ sb.append(" (approximately)");
+ }
+ sb.append(", column ").append(e.charPositionInLine);
+ sb.append(". Query: ").append(e.input.toString());
+
+ throw new AttributeExpressionLanguageParsingException(sb.toString());
+ }
+
+ public void recover(RecognitionException e) {
+ final StringBuilder sb = new StringBuilder();
+ if ( e.token == null ) {
+ sb.append("Unrecognized token ");
+ } else {
+ sb.append("Unexpected token '").append(e.token.getText()).append("' ");
+ }
+ sb.append("at line ").append(e.line);
+ if ( e.approximateLineInfo ) {
+ sb.append(" (approximately)");
+ }
+ sb.append(", column ").append(e.charPositionInLine);
+ sb.append(". Query: ").append(e.input.toString());
+
+ throw new AttributeExpressionLanguageParsingException(sb.toString());
+ }
+}
+
+
+// PUNCTUATION & SPECIAL CHARACTERS
+WHITESPACE : (' '|'\t'|'\n'|'\r')+ { $channel = HIDDEN; };
+COMMENT : '#' ( ~('\n') )* '\n' { $channel = HIDDEN; };
+
+DOLLAR : '$';
+LPAREN : '(';
+RPAREN : ')';
+LBRACE : '{';
+RBRACE : '}';
+COLON : ':';
+COMMA : ',';
+DOT : '.';
+SEMICOLON : ';';
+NUMBER : ('0'..'9')+;
+
+TRUE : 'true';
+FALSE : 'false';
+
+//
+// FUNCTION NAMES
+//
+
+// ATTRIBUTE KEY SELECTION FUNCTIONS
+ANY_ATTRIBUTE : 'anyAttribute';
+ANY_MATCHING_ATTRIBUTE : 'anyMatchingAttribute';
+ALL_ATTRIBUTES : 'allAttributes';
+ALL_MATCHING_ATTRIBUTES : 'allMatchingAttributes';
+ANY_DELINEATED_VALUE : 'anyDelineatedValue';
+ALL_DELINEATED_VALUES : 'allDelineatedValues';
+
+// NO-SUBJECT FUNCTIONS
+NEXT_INT : 'nextInt';
+IP : 'ip';
+UUID : 'UUID';
+HOSTNAME : 'hostname'; // requires boolean arg: prefer FQDN
+NOW : 'now';
+
+
+// 0 arg functions
+TO_UPPER : 'toUpper';
+TO_LOWER : 'toLower';
+TO_STRING : 'toString';
+LENGTH : 'length';
+TRIM : 'trim';
+IS_NULL : 'isNull';
+IS_EMPTY : 'isEmpty';
+NOT_NULL : 'notNull';
+TO_NUMBER : 'toNumber';
+URL_ENCODE : 'urlEncode';
+URL_DECODE : 'urlDecode';
+NOT : 'not';
+COUNT : 'count';
+
+// 1 arg functions
+SUBSTRING_AFTER : 'substringAfter';
+SUBSTRING_BEFORE : 'substringBefore';
+SUBSTRING_AFTER_LAST : 'substringAfterLast';
+SUBSTRING_BEFORE_LAST : 'substringBeforeLast';
+STARTS_WITH : 'startsWith';
+ENDS_WITH : 'endsWith';
+CONTAINS : 'contains';
+PREPEND : 'prepend';
+APPEND : 'append';
+INDEX_OF : 'indexOf';
+LAST_INDEX_OF : 'lastIndexOf';
+REPLACE_NULL : 'replaceNull';
+REPLACE_EMPTY : 'replaceEmpty';
+FIND : 'find'; // regex
+MATCHES : 'matches'; // regex
+EQUALS : 'equals';
+EQUALS_IGNORE_CASE : 'equalsIgnoreCase';
+GREATER_THAN : 'gt';
+LESS_THAN : 'lt';
+GREATER_THAN_OR_EQUAL : 'ge';
+LESS_THAN_OR_EQUAL : 'le';
+FORMAT : 'format'; // takes string date format; uses SimpleDateFormat
+TO_DATE : 'toDate'; // takes string date format; converts the subject to a Long based on the date format
+MOD : 'mod';
+PLUS : 'plus';
+MINUS : 'minus';
+MULTIPLY : 'multiply';
+DIVIDE : 'divide';
+TO_RADIX : 'toRadix';
+OR : 'or';
+AND : 'and';
+JOIN : 'join';
+TO_LITERAL : 'literal';
+
+// 2 arg functions
+SUBSTRING : 'substring';
+REPLACE : 'replace';
+REPLACE_ALL : 'replaceAll';
+
+
+// STRINGS
+STRING_LITERAL
+@init{StringBuilder lBuf = new StringBuilder();}
+ :
+ (
+ '"'
+ (
+ escaped=ESC {lBuf.append(getText());} |
+ normal = ~( '"' | '\\' | '\n' | '\r' | '\t' ) { lBuf.appendCodePoint(normal);}
+ )*
+ '"'
+ )
+ {
+ setText(lBuf.toString());
+ }
+ |
+ (
+ '\''
+ (
+ escaped=ESC {lBuf.append(getText());} |
+ normal = ~( '\'' | '\\' | '\n' | '\r' | '\t' ) { lBuf.appendCodePoint(normal);}
+ )*
+ '\''
+ )
+ {
+ setText(lBuf.toString());
+ }
+ ;
+
+
+fragment
+ESC
+ : '\\'
+ (
+ '"' { setText("\""); }
+ | '\'' { setText("\'"); }
+ | 'r' { setText("\r"); }
+ | 'n' { setText("\n"); }
+ | 't' { setText("\t"); }
+ | '\\' { setText("\\\\"); }
+ | nextChar = ~('"' | '\'' | 'r' | 'n' | 't' | '\\')
+ {
+ StringBuilder lBuf = new StringBuilder(); lBuf.append("\\\\").appendCodePoint(nextChar); setText(lBuf.toString());
+ }
+ )
+ ;
+
+ATTRIBUTE_NAME : (
+ ~('$' | '{' | '}' | '(' | ')' | '[' | ']' | ',' | ':' | ';' | '/' | '*' | '\'' | ' ' | '\t' | '\r' | '\n' | '0'..'9')
+ ~('$' | '{' | '}' | '(' | ')' | '[' | ']' | ',' | ':' | ';' | '/' | '*' | '\'' | ' ' | '\t' | '\r' | '\n')*
+ );
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionParser.g
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionParser.g b/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionParser.g
new file mode 100644
index 0000000..7c37530
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionParser.g
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+parser grammar AttributeExpressionParser;
+
+options {
+ output=AST;
+ tokenVocab=AttributeExpressionLexer;
+}
+
+tokens {
+ QUERY;
+ ATTRIBUTE_REFERENCE;
+ ATTR_NAME;
+ FUNCTION_CALL;
+ EXPRESSION;
+ MULTI_ATTRIBUTE_REFERENCE;
+ QUOTED_ATTR_NAME;
+}
+
+@header {
+ package org.apache.nifi.attribute.expression.language.antlr;
+ import org.apache.nifi.attribute.expression.language.exception.AttributeExpressionLanguageParsingException;
+}
+
+@members {
+ public void displayRecognitionError(String[] tokenNames, RecognitionException e) {
+ final StringBuilder sb = new StringBuilder();
+ if ( e.token == null ) {
+ sb.append("Unrecognized token ");
+ } else {
+ sb.append("Unexpected token '").append(e.token.getText()).append("' ");
+ }
+ sb.append("at line ").append(e.line);
+ if ( e.approximateLineInfo ) {
+ sb.append(" (approximately)");
+ }
+ sb.append(", column ").append(e.charPositionInLine);
+ sb.append(". Query: ").append(e.input.toString());
+
+ throw new AttributeExpressionLanguageParsingException(sb.toString());
+ }
+
+ public void recover(final RecognitionException e) {
+ final StringBuilder sb = new StringBuilder();
+ if ( e.token == null ) {
+ sb.append("Unrecognized token ");
+ } else {
+ sb.append("Unexpected token '").append(e.token.getText()).append("' ");
+ }
+ sb.append("at line ").append(e.line);
+ if ( e.approximateLineInfo ) {
+ sb.append(" (approximately)");
+ }
+ sb.append(", column ").append(e.charPositionInLine);
+ sb.append(". Query: ").append(e.input.toString());
+
+ throw new AttributeExpressionLanguageParsingException(sb.toString());
+ }
+}
+
+// functions that return Strings
+zeroArgString : (TO_UPPER | TO_LOWER | TRIM | TO_STRING | URL_ENCODE | URL_DECODE) LPAREN! RPAREN!;
+oneArgString : ((SUBSTRING_BEFORE | SUBSTRING_BEFORE_LAST | SUBSTRING_AFTER | SUBSTRING_AFTER_LAST | REPLACE_NULL | REPLACE_EMPTY |
+ PREPEND | APPEND | FORMAT | STARTS_WITH | ENDS_WITH | CONTAINS | JOIN) LPAREN! anyArg RPAREN!) |
+ (TO_RADIX LPAREN! anyArg (COMMA! anyArg)? RPAREN!);
+twoArgString : ((REPLACE | REPLACE_ALL) LPAREN! anyArg COMMA! anyArg RPAREN!) |
+ (SUBSTRING LPAREN! anyArg (COMMA! anyArg)? RPAREN!);
+
+
+// functions that return Booleans
+zeroArgBool : (IS_NULL | NOT_NULL | IS_EMPTY | NOT) LPAREN! RPAREN!;
+oneArgBool : ((FIND | MATCHES | EQUALS_IGNORE_CASE) LPAREN! anyArg RPAREN!) |
+ (GREATER_THAN | LESS_THAN | GREATER_THAN_OR_EQUAL | LESS_THAN_OR_EQUAL) LPAREN! anyArg RPAREN! |
+ (EQUALS) LPAREN! anyArg RPAREN! |
+ (AND | OR) LPAREN! anyArg RPAREN!;
+
+
+// functions that return Numbers
+zeroArgNum : (LENGTH | TO_NUMBER | COUNT) LPAREN! RPAREN!;
+oneArgNum : ((INDEX_OF | LAST_INDEX_OF) LPAREN! anyArg RPAREN!) |
+ (TO_DATE LPAREN! anyArg? RPAREN!) |
+ ((MOD | PLUS | MINUS | MULTIPLY | DIVIDE) LPAREN! anyArg RPAREN!);
+
+stringFunctionRef : zeroArgString | oneArgString | twoArgString;
+booleanFunctionRef : zeroArgBool | oneArgBool;
+numberFunctionRef : zeroArgNum | oneArgNum;
+
+anyArg : NUMBER | numberFunctionRef | STRING_LITERAL | zeroArgString | oneArgString | twoArgString | booleanLiteral | zeroArgBool | oneArgBool | expression;
+stringArg : STRING_LITERAL | zeroArgString | oneArgString | twoArgString | expression;
+functionRef : stringFunctionRef | booleanFunctionRef | numberFunctionRef;
+
+
+
+// Attribute Reference
+subject : attrName | expression;
+attrName : singleAttrName | multiAttrName;
+
+singleAttrRef : ATTRIBUTE_NAME | STRING_LITERAL;
+singleAttrName : singleAttrRef ->
+ ^(ATTR_NAME singleAttrRef);
+
+
+multiAttrFunction : ANY_ATTRIBUTE | ANY_MATCHING_ATTRIBUTE | ALL_ATTRIBUTES | ALL_MATCHING_ATTRIBUTES | ANY_DELINEATED_VALUE | ALL_DELINEATED_VALUES;
+multiAttrName : multiAttrFunction LPAREN stringArg (COMMA stringArg)* RPAREN ->
+ ^(MULTI_ATTRIBUTE_REFERENCE multiAttrFunction stringArg*);
+
+attributeRef : subject ->
+ ^(ATTRIBUTE_REFERENCE subject);
+
+
+functionCall : functionRef ->
+ ^(FUNCTION_CALL functionRef);
+
+booleanLiteral : TRUE | FALSE;
+zeroArgStandaloneFunction : (IP | UUID | NOW | NEXT_INT | HOSTNAME) LPAREN! RPAREN!;
+oneArgStandaloneFunction : (TO_LITERAL^ LPAREN! anyArg RPAREN!) |
+ (HOSTNAME^ LPAREN! booleanLiteral RPAREN!);
+standaloneFunction : zeroArgStandaloneFunction | oneArgStandaloneFunction;
+
+attributeRefOrFunctionCall : (attributeRef | standaloneFunction);
+
+expression : DOLLAR LBRACE attributeRefOrFunctionCall (COLON functionCall)* RBRACE ->
+ ^(EXPRESSION attributeRefOrFunctionCall functionCall*);
+
+query : expression EOF ->
+ ^(QUERY expression);
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/output/AttributeExpressionLexer.tokens
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/output/AttributeExpressionLexer.tokens b/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/output/AttributeExpressionLexer.tokens
new file mode 100755
index 0000000..0265bfb
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/output/AttributeExpressionLexer.tokens
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ALL_ATTRIBUTES=4
+ALL_DELINEATED_VALUES=5
+ALL_MATCHING_ATTRIBUTES=6
+AND=7
+ANY_ATTRIBUTE=8
+ANY_DELINEATED_VALUE=9
+ANY_MATCHING_ATTRIBUTE=10
+APPEND=11
+ATTRIBUTE_NAME=12
+CEIL=13
+COLON=14
+COMMA=15
+CONTAINS=16
+DIVIDE=17
+DOLLAR=18
+DOT=19
+ENDS_WITH=20
+EQUALS=21
+EQUALS_IGNORE_CASE=22
+FALSE=23
+FIND=24
+FLOOR=25
+FORMAT=26
+GREATER_THAN=27
+GREATER_THAN_OR_EQUAL=28
+HOSTNAME=29
+INDEX_OF=30
+IP=31
+IS_NULL=32
+LAST_INDEX_OF=33
+LBRACE=34
+LENGTH=35
+LESS_THAN=36
+LESS_THAN_OR_EQUAL=37
+LPAREN=38
+MATCHES=39
+MINUS=40
+MOD=41
+MULTIPLY=42
+NEXT_INT=43
+NOT=44
+NOT_NULL=45
+NOW=46
+NUMBER=47
+OR=48
+PLUS=49
+PREPEND=50
+RBRACE=51
+REPLACE=52
+REPLACE_ALL=53
+REPLACE_NULL=54
+RPAREN=55
+SEMICOLON=56
+STARTS_WITH=57
+STRING_LITERAL=58
+SUBSTRING=59
+SUBSTRING_AFTER=60
+SUBSTRING_AFTER_LAST=61
+SUBSTRING_BEFORE=62
+SUBSTRING_BEFORE_LAST=63
+TO_DATE=64
+TO_LOWER=65
+TO_NUMBER=66
+TO_RADIX=67
+TO_STRING=68
+TO_UPPER=69
+TRIM=70
+TRUE=71
+URL_DECODE=72
+URL_ENCODE=73
+UUID=74
+WHITESPACE=75
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/EmptyPreparedQuery.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/EmptyPreparedQuery.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/EmptyPreparedQuery.java
new file mode 100644
index 0000000..81da47e
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/EmptyPreparedQuery.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language;
+
+import java.util.Map;
+
+import org.apache.nifi.expression.AttributeValueDecorator;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.exception.ProcessException;
+
+public class EmptyPreparedQuery implements PreparedQuery {
+
+ private final String value;
+
+ EmptyPreparedQuery(final String value) {
+ this.value = value;
+ }
+
+ @Override
+ public String evaluateExpressions(final FlowFile flowFile, final AttributeValueDecorator decorator) throws ProcessException {
+ return value;
+ }
+
+ @Override
+ public String evaluateExpressions() throws ProcessException {
+ return value;
+ }
+
+ @Override
+ public String evaluateExpressions(final AttributeValueDecorator decorator) throws ProcessException {
+ return value;
+ }
+
+ @Override
+ public String evaluateExpressions(final FlowFile flowFile) throws ProcessException {
+ return value;
+ }
+
+ @Override
+ public String evaluateExpressions(Map<String, String> attributes) throws ProcessException {
+ return value;
+ }
+
+ @Override
+ public String evaluateExpressions(Map<String, String> attributes, AttributeValueDecorator decorator) throws ProcessException {
+ return value;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/InvalidPreparedQuery.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/InvalidPreparedQuery.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/InvalidPreparedQuery.java
new file mode 100644
index 0000000..a29e792
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/InvalidPreparedQuery.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.exception.AttributeExpressionLanguageException;
+import org.apache.nifi.expression.AttributeValueDecorator;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.exception.ProcessException;
+
+/**
+ * An implementation of PreparedQuery that throws an
+ * {@link AttributeExpressionLanguageException} when attempting to evaluate the
+ * query. This allows a PreparedQuery to be created, even though it can't be
+ * evaluated.
+ */
+public class InvalidPreparedQuery implements PreparedQuery {
+
+ private final String query;
+ private final String explanation;
+
+ public InvalidPreparedQuery(final String query, final String explanation) {
+ this.query = query;
+ this.explanation = explanation;
+ }
+
+ @Override
+ public String evaluateExpressions(final FlowFile flowFile, final AttributeValueDecorator decorator) throws ProcessException {
+ throw new AttributeExpressionLanguageException("Invalid Expression: " + query + " due to " + explanation);
+ }
+
+ @Override
+ public String evaluateExpressions() throws ProcessException {
+ throw new AttributeExpressionLanguageException("Invalid Expression: " + query + " due to " + explanation);
+ }
+
+ @Override
+ public String evaluateExpressions(final AttributeValueDecorator decorator) throws ProcessException {
+ throw new AttributeExpressionLanguageException("Invalid Expression: " + query + " due to " + explanation);
+ }
+
+ @Override
+ public String evaluateExpressions(final FlowFile flowFile) throws ProcessException {
+ throw new AttributeExpressionLanguageException("Invalid Expression: " + query + " due to " + explanation);
+ }
+
+ @Override
+ public String evaluateExpressions(final Map<String, String> attributes) throws ProcessException {
+ throw new AttributeExpressionLanguageException("Invalid Expression: " + query + " due to " + explanation);
+ }
+
+ @Override
+ public String evaluateExpressions(final Map<String, String> attributes, final AttributeValueDecorator decorator) throws ProcessException {
+ throw new AttributeExpressionLanguageException("Invalid Expression: " + query + " due to " + explanation);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/PreparedQuery.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/PreparedQuery.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/PreparedQuery.java
new file mode 100644
index 0000000..0d1b2c7
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/PreparedQuery.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language;
+
+import java.util.Map;
+
+import org.apache.nifi.expression.AttributeValueDecorator;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.exception.ProcessException;
+
+public interface PreparedQuery {
+
+ String evaluateExpressions(FlowFile flowFile, AttributeValueDecorator decorator) throws ProcessException;
+
+ String evaluateExpressions() throws ProcessException;
+
+ String evaluateExpressions(AttributeValueDecorator decorator) throws ProcessException;
+
+ String evaluateExpressions(FlowFile flowFile) throws ProcessException;
+
+ String evaluateExpressions(Map<String, String> attributes) throws ProcessException;
+
+ String evaluateExpressions(Map<String, String> attributes, AttributeValueDecorator decorator) throws ProcessException;
+
+}
[06/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/AuditDataSourceFactoryBean.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/AuditDataSourceFactoryBean.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/AuditDataSourceFactoryBean.java
new file mode 100644
index 0000000..aeb2755
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/AuditDataSourceFactoryBean.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin;
+
+import java.io.File;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import org.apache.commons.lang3.StringUtils;
+import org.h2.jdbcx.JdbcConnectionPool;
+import org.apache.nifi.util.NiFiProperties;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.FactoryBean;
+
+/**
+ *
+ */
+public class AuditDataSourceFactoryBean implements FactoryBean {
+
+ private static final Logger logger = LoggerFactory.getLogger(AuditDataSourceFactoryBean.class);
+ private static final String NF_USERNAME_PASSWORD = "nf";
+ private static final int MAX_CONNECTIONS = 5;
+
+ // database file name
+ private static final String AUDIT_DATABASE_FILE_NAME = "nifi-audit";
+
+ // ------------
+ // action table
+ // ------------
+ private static final String CREATE_ACTION_TABLE = "CREATE TABLE ACTION ("
+ + "ID INT NOT NULL PRIMARY KEY AUTO_INCREMENT, "
+ + "USER_DN VARCHAR2(255) NOT NULL, "
+ + "USER_NAME VARCHAR2(100) NOT NULL, "
+ + "SOURCE_ID VARCHAR2(100) NOT NULL, "
+ + "SOURCE_NAME VARCHAR2(1000) NOT NULL, "
+ + "SOURCE_TYPE VARCHAR2(1000) NOT NULL, "
+ + "OPERATION VARCHAR2(50) NOT NULL, "
+ + "ACTION_TIMESTAMP TIMESTAMP NOT NULL "
+ + ")";
+
+ // -----------------
+ // component details
+ // -----------------
+ private static final String CREATE_PROCESSOR_DETAILS_TABLE = "CREATE TABLE PROCESSOR_DETAILS ("
+ + "ACTION_ID INT NOT NULL PRIMARY KEY, "
+ + "TYPE VARCHAR2(1000) NOT NULL, "
+ + "FOREIGN KEY (ACTION_ID) REFERENCES ACTION(ID)"
+ + ")";
+
+ private static final String CREATE_REMOTE_PROCESS_GROUP_DETAILS_TABLE = "CREATE TABLE REMOTE_PROCESS_GROUP_DETAILS ("
+ + "ACTION_ID INT NOT NULL PRIMARY KEY, "
+ + "URI VARCHAR2(2500) NOT NULL, "
+ + "FOREIGN KEY (ACTION_ID) REFERENCES ACTION(ID)"
+ + ")";
+
+ // --------------
+ // action details
+ // --------------
+ private static final String CREATE_MOVE_DETAILS_TABLE = "CREATE TABLE MOVE_DETAILS ("
+ + "ACTION_ID INT NOT NULL PRIMARY KEY, "
+ + "GROUP_ID VARCHAR2(100) NOT NULL, "
+ + "GROUP_NAME VARCHAR2(1000) NOT NULL, "
+ + "PREVIOUS_GROUP_ID VARCHAR2(100) NOT NULL, "
+ + "PREVIOUS_GROUP_NAME VARCHAR2(1000) NOT NULL, "
+ + "FOREIGN KEY (ACTION_ID) REFERENCES ACTION(ID)"
+ + ")";
+
+ private static final String CREATE_CONFIGURE_DETAILS_TABLE = "CREATE TABLE CONFIGURE_DETAILS ("
+ + "ACTION_ID INT NOT NULL PRIMARY KEY, "
+ + "NAME VARCHAR2(1000) NOT NULL, "
+ + "VALUE VARCHAR2(5000), "
+ + "PREVIOUS_VALUE VARCHAR2(5000), "
+ + "FOREIGN KEY (ACTION_ID) REFERENCES ACTION(ID)"
+ + ")";
+
+ private static final String CREATE_CONNECT_DETAILS_TABLE = "CREATE TABLE CONNECT_DETAILS ("
+ + "ACTION_ID INT NOT NULL PRIMARY KEY, "
+ + "SOURCE_ID VARCHAR2(100) NOT NULL, "
+ + "SOURCE_NAME VARCHAR2(1000), "
+ + "SOURCE_TYPE VARCHAR2(1000) NOT NULL, "
+ + "RELATIONSHIP VARCHAR2(1000), "
+ + "DESTINATION_ID VARCHAR2(100) NOT NULL, "
+ + "DESTINATION_NAME VARCHAR2(1000), "
+ + "DESTINATION_TYPE VARCHAR2(1000) NOT NULL, "
+ + "FOREIGN KEY (ACTION_ID) REFERENCES ACTION(ID)"
+ + ")";
+
+ private static final String CREATE_PURGE_DETAILS_TABLE = "CREATE TABLE PURGE_DETAILS ("
+ + "ACTION_ID INT NOT NULL PRIMARY KEY, "
+ + "END_DATE TIMESTAMP NOT NULL, "
+ + "FOREIGN KEY (ACTION_ID) REFERENCES ACTION(ID)"
+ + ")";
+
+ private JdbcConnectionPool connectionPool;
+
+ private NiFiProperties properties;
+
+ @Override
+ public Object getObject() throws Exception {
+ if (connectionPool == null) {
+
+ // locate the repository directory
+ String repositoryDirectoryPath = properties.getProperty(NiFiProperties.REPOSITORY_DATABASE_DIRECTORY);
+
+ // ensure the repository directory is specified
+ if (repositoryDirectoryPath == null) {
+ throw new NullPointerException("Database directory must be specified.");
+ }
+
+ // create a handle to the repository directory
+ File repositoryDirectory = new File(repositoryDirectoryPath);
+
+ // get a handle to the database file
+ File databaseFile = new File(repositoryDirectory, AUDIT_DATABASE_FILE_NAME);
+
+ // format the database url
+ String databaseUrl = "jdbc:h2:" + databaseFile + ";AUTOCOMMIT=OFF;DB_CLOSE_ON_EXIT=FALSE;LOCK_MODE=3";
+ String databaseUrlAppend = properties.getProperty(NiFiProperties.H2_URL_APPEND);
+ if (StringUtils.isNotBlank(databaseUrlAppend)) {
+ databaseUrl += databaseUrlAppend;
+ }
+
+ // create the pool
+ connectionPool = JdbcConnectionPool.create(databaseUrl, NF_USERNAME_PASSWORD, NF_USERNAME_PASSWORD);
+ connectionPool.setMaxConnections(MAX_CONNECTIONS);
+
+ Connection connection = null;
+ ResultSet rs = null;
+ Statement statement = null;
+ try {
+ // get a connection
+ connection = connectionPool.getConnection();
+ connection.setAutoCommit(false);
+
+ // determine if the tables need to be created
+ rs = connection.getMetaData().getTables(null, null, "ACTION", null);
+ if (!rs.next()) {
+ logger.info("Database not built for repository: " + databaseUrl + ". Building now...");
+ RepositoryUtils.closeQuietly(rs);
+
+ // create a statement for initializing the database
+ statement = connection.createStatement();
+
+ // action table
+ statement.execute(CREATE_ACTION_TABLE);
+
+ // component details
+ statement.execute(CREATE_PROCESSOR_DETAILS_TABLE);
+ statement.execute(CREATE_REMOTE_PROCESS_GROUP_DETAILS_TABLE);
+
+ // action details
+ statement.execute(CREATE_MOVE_DETAILS_TABLE);
+ statement.execute(CREATE_CONFIGURE_DETAILS_TABLE);
+ statement.execute(CREATE_CONNECT_DETAILS_TABLE);
+ statement.execute(CREATE_PURGE_DETAILS_TABLE);
+ } else {
+ logger.info("Existing database found and connected to at: " + databaseUrl);
+ }
+
+ // commit any changes
+ connection.commit();
+ } catch (SQLException sqle) {
+ RepositoryUtils.rollback(connection, logger);
+ throw sqle;
+ } finally {
+ RepositoryUtils.closeQuietly(rs);
+ RepositoryUtils.closeQuietly(statement);
+ RepositoryUtils.closeQuietly(connection);
+ }
+ }
+
+ return connectionPool;
+ }
+
+ @Override
+ public Class getObjectType() {
+ return JdbcConnectionPool.class;
+ }
+
+ @Override
+ public boolean isSingleton() {
+ return true;
+ }
+
+ public void setProperties(NiFiProperties properties) {
+ this.properties = properties;
+ }
+
+ /**
+ * Disposes resources.
+ */
+ public void shutdown() {
+
+ // shutdown the connection pool
+ if (connectionPool != null) {
+ try {
+ connectionPool.dispose();
+ } catch (Exception e) {
+ logger.warn("Unable to dispose of connection pool: " + e.getMessage());
+ if (logger.isDebugEnabled()) {
+ logger.warn(StringUtils.EMPTY, e);
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/RepositoryUtils.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/RepositoryUtils.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/RepositoryUtils.java
new file mode 100644
index 0000000..f678d5d
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/RepositoryUtils.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import org.slf4j.Logger;
+
+/**
+ * A utility class for useful methods dealing with the repository
+ *
+ */
+public class RepositoryUtils {
+
+ public static void rollback(final Connection conn, final Logger logger) {
+ try {
+ if (null != conn) {
+ conn.rollback();
+ }
+ } catch (final SQLException sqe) {
+ logger.warn("The following problem occurred while trying to rollback " + conn + ": " + sqe.getLocalizedMessage());
+ if (logger.isDebugEnabled()) {
+ logger.debug("", sqe);
+ }
+ }
+ }
+
+ /**
+ * Closes the given statement quietly - no logging, no exceptions
+ *
+ * @param statement to close
+ */
+ public static void closeQuietly(final Statement statement) {
+
+ if (null != statement) {
+ try {
+ statement.close();
+ } catch (final SQLException se) { /*IGNORE*/
+
+ }
+ }
+ }
+
+ /**
+ * Closes the given result set quietly - no logging, no exceptions
+ *
+ * @param resultSet to close
+ */
+ public static void closeQuietly(final ResultSet resultSet) {
+ if (null != resultSet) {
+ try {
+ resultSet.close();
+ } catch (final SQLException se) {/*IGNORE*/
+
+ }
+ }
+ }
+
+ /**
+ * Closes the given connection quietly - no logging, no exceptions
+ *
+ * @param conn to close
+ */
+ public static void closeQuietly(final Connection conn) {
+ if (null != conn) {
+ try {
+ conn.close();
+ } catch (final SQLException se) {/*IGNORE*/
+
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/UserDataSourceFactoryBean.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/UserDataSourceFactoryBean.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/UserDataSourceFactoryBean.java
new file mode 100644
index 0000000..ebcf574
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/UserDataSourceFactoryBean.java
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin;
+
+import java.io.File;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.UUID;
+import org.apache.commons.lang3.StringUtils;
+import org.h2.jdbcx.JdbcConnectionPool;
+import org.apache.nifi.user.NiFiUser;
+import org.apache.nifi.util.NiFiProperties;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.FactoryBean;
+
+public class UserDataSourceFactoryBean implements FactoryBean {
+
+ private static final Logger logger = LoggerFactory.getLogger(UserDataSourceFactoryBean.class);
+ private static final String NF_USERNAME_PASSWORD = "nf";
+ private static final int MAX_CONNECTIONS = 5;
+
+ // database file name
+ private static final String AUDIT_DATABASE_FILE_NAME = "nifi-users";
+
+ private static final String CREATE_USER_TABLE = "CREATE TABLE USER ("
+ + "ID VARCHAR2(100) NOT NULL PRIMARY KEY, "
+ + "DN VARCHAR2(255) NOT NULL UNIQUE, "
+ + "USER_NAME VARCHAR2(100) NOT NULL, "
+ + "USER_GROUP VARCHAR2(100), "
+ + "CREATION TIMESTAMP NOT NULL, "
+ + "LAST_ACCESSED TIMESTAMP, "
+ + "LAST_VERIFIED TIMESTAMP, "
+ + "JUSTIFICATION VARCHAR2(500) NOT NULL, "
+ + "STATUS VARCHAR2(10) NOT NULL"
+ + ")";
+
+ private static final String CREATE_AUTHORITY_TABLE = "CREATE TABLE AUTHORITY ("
+ + "ID INT NOT NULL PRIMARY KEY AUTO_INCREMENT, "
+ + "USER_ID VARCHAR2(100) NOT NULL, "
+ + "ROLE VARCHAR2(50) NOT NULL, "
+ + "FOREIGN KEY (USER_ID) REFERENCES USER (ID), "
+ + "CONSTRAINT USER_ROLE_UNIQUE_CONSTRAINT UNIQUE (USER_ID, ROLE)"
+ + ")";
+
+ private static final String INSERT_ANONYMOUS_USER = "INSERT INTO USER ("
+ + "ID, DN, USER_NAME, CREATION, LAST_VERIFIED, JUSTIFICATION, STATUS"
+ + ") VALUES ("
+ + "'" + UUID.randomUUID().toString() + "', "
+ + "'" + NiFiUser.ANONYMOUS_USER_DN + "', "
+ + "'" + NiFiUser.ANONYMOUS_USER_DN + "', "
+ + "NOW(), "
+ + "NOW(), "
+ + "'Anonymous user needs no justification', "
+ + "'ACTIVE'"
+ + ")";
+
+ private static final String INSERT_ANONYMOUS_MONITOR_AUTHORITY = "INSERT INTO AUTHORITY ("
+ + "USER_ID, ROLE"
+ + ") VALUES ("
+ + "(SELECT ID FROM USER WHERE DN = '" + NiFiUser.ANONYMOUS_USER_DN + "'), "
+ + "'ROLE_MONITOR'"
+ + ")";
+
+ private static final String INSERT_ANONYMOUS_DFM_AUTHORITY = "INSERT INTO AUTHORITY ("
+ + "USER_ID, ROLE"
+ + ") VALUES ("
+ + "(SELECT ID FROM USER WHERE DN = '" + NiFiUser.ANONYMOUS_USER_DN + "'), "
+ + "'ROLE_DFM'"
+ + ")";
+
+ private static final String INSERT_ANONYMOUS_ADMIN_AUTHORITY = "INSERT INTO AUTHORITY ("
+ + "USER_ID, ROLE"
+ + ") VALUES ("
+ + "(SELECT ID FROM USER WHERE DN = '" + NiFiUser.ANONYMOUS_USER_DN + "'), "
+ + "'ROLE_ADMIN'"
+ + ")";
+
+ private static final String INSERT_ANONYMOUS_NIFI_AUTHORITY = "INSERT INTO AUTHORITY ("
+ + "USER_ID, ROLE"
+ + ") VALUES ("
+ + "(SELECT ID FROM USER WHERE DN = '" + NiFiUser.ANONYMOUS_USER_DN + "'), "
+ + "'ROLE_NIFI'"
+ + ")";
+
+ private static final String INSERT_ANONYMOUS_PROVENANCE_AUTHORITY = "INSERT INTO AUTHORITY ("
+ + "USER_ID, ROLE"
+ + ") VALUES ("
+ + "(SELECT ID FROM USER WHERE DN = '" + NiFiUser.ANONYMOUS_USER_DN + "'), "
+ + "'ROLE_PROVENANCE'"
+ + ")";
+
+ private static final String SELECT_ANONYMOUS_PROVENANCE_AUTHORITY = "SELECT * FROM AUTHORITY "
+ + "WHERE "
+ + "USER_ID = (SELECT ID FROM USER WHERE DN = '" + NiFiUser.ANONYMOUS_USER_DN + "') "
+ + "AND "
+ + "ROLE = 'ROLE_PROVENANCE'";
+
+ private JdbcConnectionPool connectionPool;
+
+ private NiFiProperties properties;
+
+ @Override
+ public Object getObject() throws Exception {
+ if (connectionPool == null) {
+
+ // locate the repository directory
+ String repositoryDirectoryPath = properties.getProperty(NiFiProperties.REPOSITORY_DATABASE_DIRECTORY);
+
+ // ensure the repository directory is specified
+ if (repositoryDirectoryPath == null) {
+ throw new NullPointerException("Database directory must be specified.");
+ }
+
+ // create a handle to the repository directory
+ File repositoryDirectory = new File(repositoryDirectoryPath);
+
+ // create a handle to the database directory and file
+ File databaseFile = new File(repositoryDirectory, AUDIT_DATABASE_FILE_NAME);
+ String databaseUrl = getDatabaseUrl(databaseFile);
+
+ // create the pool
+ connectionPool = JdbcConnectionPool.create(databaseUrl, NF_USERNAME_PASSWORD, NF_USERNAME_PASSWORD);
+ connectionPool.setMaxConnections(MAX_CONNECTIONS);
+
+ Connection connection = null;
+ ResultSet rs = null;
+ Statement statement = null;
+ try {
+ // get a connection
+ connection = connectionPool.getConnection();
+ connection.setAutoCommit(false);
+
+ // create a statement for creating/updating the database
+ statement = connection.createStatement();
+
+ // determine if the tables need to be created
+ rs = connection.getMetaData().getTables(null, null, "USER", null);
+ if (!rs.next()) {
+ logger.info("Database not built for repository: " + databaseUrl + ". Building now...");
+
+ // create the tables
+ statement.execute(CREATE_USER_TABLE);
+ statement.execute(CREATE_AUTHORITY_TABLE);
+
+ // seed the anonymous user
+ statement.execute(INSERT_ANONYMOUS_USER);
+ statement.execute(INSERT_ANONYMOUS_MONITOR_AUTHORITY);
+ statement.execute(INSERT_ANONYMOUS_DFM_AUTHORITY);
+ statement.execute(INSERT_ANONYMOUS_ADMIN_AUTHORITY);
+ statement.execute(INSERT_ANONYMOUS_NIFI_AUTHORITY);
+ } else {
+ logger.info("Existing database found and connected to at: " + databaseUrl);
+ }
+
+ // close the previous result set
+ RepositoryUtils.closeQuietly(rs);
+
+ // merge in the provenance role to handle existing databases
+ rs = statement.executeQuery(SELECT_ANONYMOUS_PROVENANCE_AUTHORITY);
+ if (!rs.next()) {
+ statement.execute(INSERT_ANONYMOUS_PROVENANCE_AUTHORITY);
+ }
+
+ // commit any changes
+ connection.commit();
+ } catch (SQLException sqle) {
+ RepositoryUtils.rollback(connection, logger);
+ throw sqle;
+ } finally {
+ RepositoryUtils.closeQuietly(rs);
+ RepositoryUtils.closeQuietly(statement);
+ RepositoryUtils.closeQuietly(connection);
+ }
+ }
+
+ return connectionPool;
+ }
+
+ private String getDatabaseUrl(File databaseFile) {
+ String databaseUrl = "jdbc:h2:" + databaseFile + ";AUTOCOMMIT=OFF;DB_CLOSE_ON_EXIT=FALSE;LOCK_MODE=3";
+ String databaseUrlAppend = properties.getProperty(NiFiProperties.H2_URL_APPEND);
+ if (StringUtils.isNotBlank(databaseUrlAppend)) {
+ databaseUrl += databaseUrlAppend;
+ }
+ return databaseUrl;
+ }
+
+ @Override
+ public Class getObjectType() {
+ return JdbcConnectionPool.class;
+ }
+
+ @Override
+ public boolean isSingleton() {
+ return true;
+ }
+
+ public void setProperties(NiFiProperties properties) {
+ this.properties = properties;
+ }
+
+ public void shutdown() {
+
+ // shutdown the connection pool
+ if (connectionPool != null) {
+ try {
+ connectionPool.dispose();
+ } catch (Exception e) {
+ logger.warn("Unable to dispose of connection pool: " + e.getMessage());
+ if (logger.isDebugEnabled()) {
+ logger.warn(StringUtils.EMPTY, e);
+ }
+ }
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/ActionDAO.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/ActionDAO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/ActionDAO.java
new file mode 100644
index 0000000..925dc80
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/ActionDAO.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.dao;
+
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import org.apache.nifi.action.Action;
+import org.apache.nifi.history.HistoryQuery;
+import org.apache.nifi.history.History;
+import org.apache.nifi.history.PreviousValue;
+
+/**
+ * Action data access.
+ */
+public interface ActionDAO {
+
+ /**
+ * Persists the specified action.
+ *
+ * @param action to persist
+ * @throws DataAccessException if unable to persist
+ */
+ void createAction(Action action) throws DataAccessException;
+
+ /**
+ * Finds all actions that meet the specified criteria.
+ *
+ * @param actionQuery query for actions
+ * @return History of actions
+ * @throws DataAccessException dae
+ */
+ History findActions(HistoryQuery actionQuery) throws DataAccessException;
+
+ /**
+ * @param componentId to get previous values of
+ * @return Finds the previous values for the specified property in the
+ * specified component. Returns empty list if there are none
+ */
+ Map<String, List<PreviousValue>> getPreviousValues(String componentId);
+
+ /**
+ * Finds the specified action.
+ *
+ * @param actionId action identifier
+ * @return Action specified
+ * @throws DataAccessException dae
+ */
+ Action getAction(Integer actionId) throws DataAccessException;
+
+ /**
+ * Deletes all actions up to the specified end date.
+ *
+ * @param endDate date to stop deleting at
+ * @throws DataAccessException dae
+ */
+ void deleteActions(Date endDate) throws DataAccessException;
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/AuthorityDAO.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/AuthorityDAO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/AuthorityDAO.java
new file mode 100644
index 0000000..b80b78e
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/AuthorityDAO.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.dao;
+
+import java.util.Set;
+import org.apache.nifi.authorization.Authority;
+
+/**
+ * Authority data access.
+ */
+public interface AuthorityDAO {
+
+ /**
+ * Finds all Authority for the specified user.
+ *
+ * @param userId identifier of user
+ * @return authorities
+ */
+ Set<Authority> findAuthoritiesByUserId(String userId) throws DataAccessException;
+
+ /**
+ * Creates new Authorities for the specified user in addition to authorities
+ * they already have.
+ *
+ * @param authorities to add to the given user
+ * @param userId identifier of user
+ */
+ void createAuthorities(Set<Authority> authorities, String userId) throws DataAccessException;
+
+ /**
+ * Removes all Authorities for the specified user.
+ *
+ * @param userId user identifier
+ * @throws DataAccessException if unable to access authorities
+ */
+ void deleteAuthorities(String userId) throws DataAccessException;
+
+ /**
+ * Removes the specified Authority.
+ *
+ * @param authorities to remove
+ * @param userId user id
+ */
+ void deleteAuthorities(Set<Authority> authorities, String userId) throws DataAccessException;
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/DAOFactory.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/DAOFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/DAOFactory.java
new file mode 100644
index 0000000..dee4ef9
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/DAOFactory.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.dao;
+
+/**
+ *
+ */
+public interface DAOFactory {
+
+ UserDAO getUserDAO();
+
+ ActionDAO getActionDAO();
+
+ AuthorityDAO getAuthorityDAO();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/DataAccessException.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/DataAccessException.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/DataAccessException.java
new file mode 100644
index 0000000..05bf4af
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/DataAccessException.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.dao;
+
+/**
+ * Represents any error that might occur while administering NiFi accounts.
+ */
+public class DataAccessException extends RuntimeException {
+
+ public DataAccessException(Throwable cause) {
+ super(cause);
+ }
+
+ public DataAccessException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public DataAccessException(String message) {
+ super(message);
+ }
+
+ public DataAccessException() {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/UserDAO.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/UserDAO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/UserDAO.java
new file mode 100644
index 0000000..6339e5a
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/UserDAO.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.dao;
+
+import java.util.Date;
+import java.util.Set;
+import org.apache.nifi.user.AccountStatus;
+import org.apache.nifi.user.NiFiUser;
+
+/**
+ * Defines the user data access object.
+ */
+public interface UserDAO {
+
+ /**
+ * Determines whether there are any PENDING user accounts.
+ *
+ * @return true if pending
+ * @throws DataAccessException dae
+ */
+ Boolean hasPendingUserAccounts() throws DataAccessException;
+
+ /**
+ * Returns all users.
+ *
+ * @return all users
+ * @throws DataAccessException dae
+ */
+ Set<NiFiUser> findUsers() throws DataAccessException;
+
+ /**
+ * Returns all user groups.
+ *
+ * @return all group names
+ * @throws DataAccessException dae
+ */
+ Set<String> findUserGroups() throws DataAccessException;
+
+ /**
+ * Returns all users for the specified group.
+ *
+ * @param group group
+ * @return users in group
+ * @throws DataAccessException dae
+ */
+ Set<NiFiUser> findUsersForGroup(String group) throws DataAccessException;
+
+ /**
+ * Returns the user with the specified id.
+ *
+ * @param id user id
+ * @return user for the given id
+ * @throws DataAccessException dae
+ */
+ NiFiUser findUserById(String id) throws DataAccessException;
+
+ /**
+ * Returns the user with the specified DN.
+ *
+ * @param dn user dn
+ * @return user
+ */
+ NiFiUser findUserByDn(String dn) throws DataAccessException;
+
+ /**
+ * Creates a new user based off the specified NiFiUser.
+ *
+ * @param user to create
+ */
+ void createUser(NiFiUser user) throws DataAccessException;
+
+ /**
+ * Updates the specified NiFiUser.
+ *
+ * @param user to update
+ */
+ void updateUser(NiFiUser user) throws DataAccessException;
+
+ /**
+ * Deletes the specified user.
+ *
+ * @param id user identifier
+ * @throws DataAccessException dae
+ */
+ void deleteUser(String id) throws DataAccessException;
+
+ /**
+ * Sets the status of the specified group.
+ *
+ * @param group group
+ * @param status status
+ * @throws DataAccessException dae
+ */
+ void updateGroupStatus(String group, AccountStatus status) throws DataAccessException;
+
+ /**
+ * Sets the last verified time for all users in the specified group.
+ *
+ * @param group group
+ * @param lastVerified date last verified
+ * @throws DataAccessException dae
+ */
+ void updateGroupVerification(String group, Date lastVerified) throws DataAccessException;
+
+ /**
+ * Ungroups the specified group.
+ *
+ * @param group to ungroup
+ * @throws DataAccessException dae
+ */
+ void ungroup(String group) throws DataAccessException;
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/impl/DAOFactoryImpl.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/impl/DAOFactoryImpl.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/impl/DAOFactoryImpl.java
new file mode 100644
index 0000000..2f3de0e
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/impl/DAOFactoryImpl.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.dao.impl;
+
+import java.sql.Connection;
+import org.apache.nifi.admin.dao.ActionDAO;
+import org.apache.nifi.admin.dao.AuthorityDAO;
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.admin.dao.UserDAO;
+
+/**
+ *
+ */
+public class DAOFactoryImpl implements DAOFactory {
+
+ private final Connection connection;
+
+ public DAOFactoryImpl(Connection connection) {
+ this.connection = connection;
+ }
+
+ @Override
+ public ActionDAO getActionDAO() {
+ return new StandardActionDAO(connection);
+ }
+
+ @Override
+ public AuthorityDAO getAuthorityDAO() {
+ return new StandardAuthorityDAO(connection);
+ }
+
+ @Override
+ public UserDAO getUserDAO() {
+ return new StandardUserDAO(connection);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/impl/StandardActionDAO.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/impl/StandardActionDAO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/impl/StandardActionDAO.java
new file mode 100644
index 0000000..bb655eb
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/impl/StandardActionDAO.java
@@ -0,0 +1,965 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.dao.impl;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.nifi.action.Action;
+import org.apache.nifi.action.Component;
+import org.apache.nifi.action.Operation;
+import org.apache.nifi.action.component.details.ComponentDetails;
+import org.apache.nifi.action.component.details.ExtensionDetails;
+import org.apache.nifi.action.component.details.RemoteProcessGroupDetails;
+import org.apache.nifi.action.details.ActionDetails;
+import org.apache.nifi.action.details.ConfigureDetails;
+import org.apache.nifi.action.details.ConnectDetails;
+import org.apache.nifi.action.details.MoveDetails;
+import org.apache.nifi.action.details.PurgeDetails;
+import org.apache.nifi.admin.RepositoryUtils;
+import org.apache.nifi.admin.dao.ActionDAO;
+import org.apache.nifi.admin.dao.DataAccessException;
+import org.apache.nifi.history.History;
+import org.apache.nifi.history.HistoryQuery;
+import org.apache.nifi.history.PreviousValue;
+import org.apache.commons.lang3.StringUtils;
+
+/**
+ *
+ */
+public class StandardActionDAO implements ActionDAO {
+
+ // ------------
+ // action table
+ // ------------
+ private static final String INSERT_ACTION = "INSERT INTO ACTION ("
+ + "USER_DN, USER_NAME, SOURCE_ID, SOURCE_NAME, SOURCE_TYPE, OPERATION, ACTION_TIMESTAMP"
+ + ") VALUES ("
+ + "?, "
+ + "?, "
+ + "?, "
+ + "?, "
+ + "?, "
+ + "?, "
+ + "?, "
+ + ")";
+
+ // -----------------
+ // component details
+ // -----------------
+ private static final String INSERT_EXTENSION_DETAILS = "INSERT INTO PROCESSOR_DETAILS ("
+ + "ACTION_ID, TYPE"
+ + ") VALUES ("
+ + "?, "
+ + "?"
+ + ")";
+
+ private static final String INSERT_REMOTE_PROCESS_GROUP_DETAILS = "INSERT INTO REMOTE_PROCESS_GROUP_DETAILS ("
+ + "ACTION_ID, URI"
+ + ") VALUES ("
+ + "?, "
+ + "?"
+ + ")";
+
+ // --------------
+ // action details
+ // --------------
+ private static final String INSERT_CONFIGURE_DETAILS = "INSERT INTO CONFIGURE_DETAILS ("
+ + "ACTION_ID, NAME, VALUE, PREVIOUS_VALUE"
+ + ") VALUES ("
+ + "?, "
+ + "?, "
+ + "?, "
+ + "?"
+ + ")";
+
+ private static final String INSERT_CONNECT_DETAILS = "INSERT INTO CONNECT_DETAILS ("
+ + "ACTION_ID, SOURCE_ID, SOURCE_NAME, SOURCE_TYPE, RELATIONSHIP, DESTINATION_ID, DESTINATION_NAME, DESTINATION_TYPE"
+ + ") VALUES ("
+ + "?, "
+ + "?, "
+ + "?, "
+ + "?, "
+ + "?, "
+ + "?, "
+ + "?, "
+ + "?"
+ + ")";
+
+ private static final String INSERT_MOVE_DETAILS = "INSERT INTO MOVE_DETAILS ("
+ + "ACTION_ID, GROUP_ID, GROUP_NAME, PREVIOUS_GROUP_ID, PREVIOUS_GROUP_NAME"
+ + ") VALUES ("
+ + "?, "
+ + "?, "
+ + "?, "
+ + "?, "
+ + "?"
+ + ")";
+
+ private static final String INSERT_PURGE_DETAILS = "INSERT INTO PURGE_DETAILS ("
+ + "ACTION_ID, END_DATE"
+ + ") VALUES ("
+ + "?, "
+ + "?"
+ + ")";
+
+ // ------------
+ // action table
+ // ------------
+ private static final String SELECT_ACTIONS = "SELECT * FROM ACTION";
+
+ private static final String SELECT_ACTION_COUNT = "SELECT COUNT(*) AS ACTION_COUNT FROM ACTION";
+
+ private static final String SELECT_ACTION_BY_ID = "SELECT * "
+ + "FROM ACTION "
+ + "WHERE "
+ + "ID = ?";
+
+ private static final String DELETE_ACTIONS = "DELETE FROM ACTION WHERE ACTION_TIMESTAMP < ?";
+
+ private static final String DELETE_SPECIFIC_ACTIONS = "DELETE FROM %s WHERE %s IN (SELECT ID FROM ACTION WHERE ACTION_TIMESTAMP < ?)";
+
+ // -----------------
+ // component details
+ // -----------------
+ private static final String SELECT_EXTENSION_DETAILS_FOR_ACTION = "SELECT * FROM PROCESSOR_DETAILS WHERE ACTION_ID = ?";
+
+ private static final String SELECT_REMOTE_PROCESS_GROUP_DETAILS_FOR_ACTION = "SELECT * FROM REMOTE_PROCESS_GROUP_DETAILS WHERE ACTION_ID = ?";
+
+ // --------------
+ // action details
+ // --------------
+ private static final String SELECT_MOVE_DETAILS_FOR_ACTION = "SELECT * FROM MOVE_DETAILS WHERE ACTION_ID = ?";
+
+ private static final String SELECT_CONFIGURE_DETAILS_FOR_ACTION = "SELECT * FROM CONFIGURE_DETAILS WHERE ACTION_ID = ?";
+
+ private static final String SELECT_CONNECT_DETAILS_FOR_ACTION = "SELECT * FROM CONNECT_DETAILS WHERE ACTION_ID = ?";
+
+ private static final String SELECT_PURGE_DETAILS_FOR_ACTION = "SELECT * FROM PURGE_DETAILS WHERE ACTION_ID = ?";
+
+ // ---------------
+ // previous values
+ // ---------------
+ private static final String SELECT_PREVIOUSLY_CONFIGURED_FIELDS = "SELECT DISTINCT CD.NAME "
+ + "FROM CONFIGURE_DETAILS CD "
+ + "INNER JOIN ACTION A "
+ + "ON CD.ACTION_ID = A.ID "
+ + "WHERE A.SOURCE_ID = ?";
+
+ private static final String SELECT_PREVIOUS_VALUES = "SELECT CD.VALUE, "
+ + "A.ACTION_TIMESTAMP, "
+ + "A.USER_NAME "
+ + "FROM CONFIGURE_DETAILS CD "
+ + "INNER JOIN ACTION A "
+ + "ON CD.ACTION_ID = A.ID "
+ + "WHERE A.SOURCE_ID = ? AND CD.NAME = ? "
+ + "ORDER BY A.ACTION_TIMESTAMP DESC "
+ + "LIMIT 4";
+
+ private final Connection connection;
+ private final Map<String, String> columnMap;
+
+ public StandardActionDAO(Connection connection) {
+ this.connection = connection;
+
+ // initialize the column mappings
+ this.columnMap = new HashMap<>();
+ this.columnMap.put("timestamp", "ACTION_TIMESTAMP");
+ this.columnMap.put("sourceName", "SOURCE_NAME");
+ this.columnMap.put("sourceType", "SOURCE_TYPE");
+ this.columnMap.put("operation", "OPERATION");
+ this.columnMap.put("userName", "USER_NAME");
+ }
+
+ @Override
+ public void createAction(Action action) throws DataAccessException {
+ if (action.getUserDn() == null) {
+ throw new IllegalArgumentException("User cannot be null.");
+ }
+
+ if (action.getTimestamp() == null) {
+ throw new IllegalArgumentException("Action timestamp cannot be null.");
+ }
+
+ PreparedStatement statement = null;
+ ResultSet rs = null;
+ try {
+ // obtain a statement to insert to the action table
+ statement = connection.prepareStatement(INSERT_ACTION, Statement.RETURN_GENERATED_KEYS);
+ statement.setString(1, StringUtils.left(action.getUserDn(), 255));
+ statement.setString(2, StringUtils.left(action.getUserName(), 100));
+ statement.setString(3, action.getSourceId());
+ statement.setString(4, StringUtils.left(action.getSourceName(), 1000));
+ statement.setString(5, action.getSourceType().toString());
+ statement.setString(6, action.getOperation().toString());
+ statement.setTimestamp(7, new java.sql.Timestamp(action.getTimestamp().getTime()));
+
+ // insert the action
+ int updateCount = statement.executeUpdate();
+
+ // get the action id
+ rs = statement.getGeneratedKeys();
+ if (updateCount == 1 && rs.next()) {
+ action.setId(rs.getInt(1));
+ } else {
+ throw new DataAccessException("Unable to insert action.");
+ }
+
+ // close the previous statement
+ statement.close();
+
+ // determine the type of component
+ ComponentDetails componentDetails = action.getComponentDetails();
+ if (componentDetails instanceof ExtensionDetails) {
+ createExtensionDetails(action.getId(), (ExtensionDetails) componentDetails);
+ } else if (componentDetails instanceof RemoteProcessGroupDetails) {
+ createRemoteProcessGroupDetails(action.getId(), (RemoteProcessGroupDetails) componentDetails);
+ }
+
+ // determine the type of action
+ ActionDetails details = action.getActionDetails();
+ if (details instanceof ConnectDetails) {
+ createConnectDetails(action.getId(), (ConnectDetails) details);
+ } else if (details instanceof MoveDetails) {
+ createMoveDetails(action.getId(), (MoveDetails) details);
+ } else if (details instanceof ConfigureDetails) {
+ createConfigureDetails(action.getId(), (ConfigureDetails) details);
+ } else if (details instanceof PurgeDetails) {
+ createPurgeDetails(action.getId(), (PurgeDetails) details);
+ }
+
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } finally {
+ RepositoryUtils.closeQuietly(rs);
+ RepositoryUtils.closeQuietly(statement);
+ }
+ }
+
+ private void createExtensionDetails(int actionId, ExtensionDetails extensionDetails) throws DataAccessException {
+ PreparedStatement statement = null;
+ try {
+ // obtain a statement to insert to the extension action table
+ statement = connection.prepareStatement(INSERT_EXTENSION_DETAILS);
+ statement.setInt(1, actionId);
+ statement.setString(2, StringUtils.left(extensionDetails.getType(), 1000));
+
+ // insert the action
+ int updateCount = statement.executeUpdate();
+
+ // ensure the operation completed successfully
+ if (updateCount != 1) {
+ throw new DataAccessException("Unable to insert extension details.");
+ }
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } finally {
+ RepositoryUtils.closeQuietly(statement);
+ }
+ }
+
+ private void createRemoteProcessGroupDetails(int actionId, RemoteProcessGroupDetails remoteProcessGroupDetails) throws DataAccessException {
+ PreparedStatement statement = null;
+ try {
+ // obtain a statement to insert to the processor action table
+ statement = connection.prepareStatement(INSERT_REMOTE_PROCESS_GROUP_DETAILS);
+ statement.setInt(1, actionId);
+ statement.setString(2, StringUtils.left(remoteProcessGroupDetails.getUri(), 2500));
+
+ // insert the action
+ int updateCount = statement.executeUpdate();
+
+ // ensure the operation completed successfully
+ if (updateCount != 1) {
+ throw new DataAccessException("Unable to insert remote prcoess group details.");
+ }
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } finally {
+ RepositoryUtils.closeQuietly(statement);
+ }
+ }
+
+ private void createConnectDetails(int actionId, ConnectDetails connectionDetails) throws DataAccessException {
+ PreparedStatement statement = null;
+ try {
+ // obtain a statement to insert to the processor action table
+ statement = connection.prepareStatement(INSERT_CONNECT_DETAILS);
+ statement.setInt(1, actionId);
+ statement.setString(2, connectionDetails.getSourceId());
+ statement.setString(3, StringUtils.left(connectionDetails.getSourceName(), 1000));
+ statement.setString(4, StringUtils.left(connectionDetails.getSourceType().toString(), 1000));
+ statement.setString(5, StringUtils.left(connectionDetails.getRelationship(), 1000));
+ statement.setString(6, connectionDetails.getDestinationId());
+ statement.setString(7, StringUtils.left(connectionDetails.getDestinationName(), 1000));
+ statement.setString(8, StringUtils.left(connectionDetails.getDestinationType().toString(), 1000));
+
+ // insert the action
+ int updateCount = statement.executeUpdate();
+
+ // ensure the operation completed successfully
+ if (updateCount != 1) {
+ throw new DataAccessException("Unable to insert connection details.");
+ }
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } finally {
+ RepositoryUtils.closeQuietly(statement);
+ }
+ }
+
+ private void createMoveDetails(int actionId, MoveDetails moveDetails) throws DataAccessException {
+ PreparedStatement statement = null;
+ try {
+ // obtain a statement to insert to the processor action table
+ statement = connection.prepareStatement(INSERT_MOVE_DETAILS);
+ statement.setInt(1, actionId);
+ statement.setString(2, moveDetails.getGroupId());
+ statement.setString(3, StringUtils.left(moveDetails.getGroup(), 1000));
+ statement.setString(4, moveDetails.getPreviousGroupId());
+ statement.setString(5, StringUtils.left(moveDetails.getPreviousGroup(), 1000));
+
+ // insert the action
+ int updateCount = statement.executeUpdate();
+
+ // ensure the operation completed successfully
+ if (updateCount != 1) {
+ throw new DataAccessException("Unable to insert move details.");
+ }
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } finally {
+ RepositoryUtils.closeQuietly(statement);
+ }
+ }
+
+ private void createConfigureDetails(int actionId, ConfigureDetails configurationDetails) throws DataAccessException {
+ PreparedStatement statement = null;
+ try {
+ // obtain a statement to insert to the processor action table
+ statement = connection.prepareStatement(INSERT_CONFIGURE_DETAILS);
+ statement.setInt(1, actionId);
+ statement.setString(2, StringUtils.left(configurationDetails.getName(), 1000));
+ statement.setString(3, StringUtils.left(configurationDetails.getValue(), 5000));
+ statement.setString(4, StringUtils.left(configurationDetails.getPreviousValue(), 5000));
+
+ // insert the action
+ int updateCount = statement.executeUpdate();
+
+ // ensure the operation completed successfully
+ if (updateCount != 1) {
+ throw new DataAccessException("Unable to insert configure details.");
+ }
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } finally {
+ RepositoryUtils.closeQuietly(statement);
+ }
+ }
+
+ private void createPurgeDetails(int actionId, PurgeDetails purgeDetails) throws DataAccessException {
+ PreparedStatement statement = null;
+ try {
+ // obtain a statement to insert to the processor action table
+ statement = connection.prepareStatement(INSERT_PURGE_DETAILS);
+ statement.setInt(1, actionId);
+ statement.setTimestamp(2, new java.sql.Timestamp(purgeDetails.getEndDate().getTime()));
+
+ // insert the action
+ int updateCount = statement.executeUpdate();
+
+ // ensure the operation completed successfully
+ if (updateCount != 1) {
+ throw new DataAccessException("Unable to insert connection details.");
+ }
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } finally {
+ RepositoryUtils.closeQuietly(statement);
+ }
+ }
+
+ @Override
+ public History findActions(HistoryQuery historyQuery) throws DataAccessException {
+
+ // get the sort column
+ String sortColumn = "ACTION_TIMESTAMP";
+ if (StringUtils.isNotBlank(historyQuery.getSortColumn())) {
+ String rawColumnName = historyQuery.getSortColumn();
+ if (!columnMap.containsKey(rawColumnName)) {
+ throw new IllegalArgumentException(String.format("Unrecognized column name '%s'.", rawColumnName));
+ }
+ sortColumn = columnMap.get(rawColumnName);
+ }
+
+ // get the sort order
+ String sortOrder = "desc";
+ if (StringUtils.isNotBlank(historyQuery.getSortOrder())) {
+ sortOrder = historyQuery.getSortOrder();
+ }
+
+ History actionResult = new History();
+ Collection<Action> actions = new ArrayList<>();
+ PreparedStatement statement = null;
+ ResultSet rs = null;
+ try {
+ List<String> where = new ArrayList<>();
+
+ // append the start time
+ if (historyQuery.getStartDate() != null) {
+ where.add("ACTION_TIMESTAMP >= ?");
+ }
+
+ // append the end time
+ if (historyQuery.getEndDate() != null) {
+ where.add("ACTION_TIMESTAMP <= ?");
+ }
+
+ // append the user id as necessary
+ if (historyQuery.getUserName() != null) {
+ where.add("UPPER(USER_NAME) LIKE ?");
+ }
+
+ // append the source id as necessary
+ if (historyQuery.getSourceId() != null) {
+ where.add("SOURCE_ID = ?");
+ }
+
+ String sql = SELECT_ACTION_COUNT;
+ if (!where.isEmpty()) {
+ sql += " WHERE " + StringUtils.join(where, " AND ");
+ }
+
+ // get the total number of actions
+ statement = connection.prepareStatement(sql);
+ int paramIndex = 1;
+
+ // set the start date as necessary
+ if (historyQuery.getStartDate() != null) {
+ statement.setTimestamp(paramIndex++, new java.sql.Timestamp(historyQuery.getStartDate().getTime()));
+ }
+
+ // set the end date as necessary
+ if (historyQuery.getEndDate() != null) {
+ statement.setTimestamp(paramIndex++, new java.sql.Timestamp(historyQuery.getEndDate().getTime()));
+ }
+
+ // set the user id as necessary
+ if (historyQuery.getUserName() != null) {
+ statement.setString(paramIndex++, "%" + historyQuery.getUserName().toUpperCase() + "%");
+ }
+
+ // set the source id as necessary
+ if (historyQuery.getSourceId() != null) {
+ statement.setString(paramIndex, historyQuery.getSourceId());
+ }
+
+ // execute the statement
+ rs = statement.executeQuery();
+
+ // ensure there are results
+ if (rs.next()) {
+ actionResult.setTotal(rs.getInt("ACTION_COUNT"));
+ } else {
+ throw new DataAccessException("Unable to determine total action count.");
+ }
+
+ sql = SELECT_ACTIONS;
+ if (!where.isEmpty()) {
+ sql += " WHERE " + StringUtils.join(where, " AND ");
+ }
+
+ // append the sort criteria
+ sql += (" ORDER BY " + sortColumn + " " + sortOrder);
+
+ // append the offset and limit
+ sql += " LIMIT ? OFFSET ?";
+
+ // close the previous statement
+ statement.close();
+
+ // create the statement
+ statement = connection.prepareStatement(sql);
+ paramIndex = 1;
+
+ // set the start date as necessary
+ if (historyQuery.getStartDate() != null) {
+ statement.setTimestamp(paramIndex++, new java.sql.Timestamp(historyQuery.getStartDate().getTime()));
+ }
+
+ // set the end date as necessary
+ if (historyQuery.getEndDate() != null) {
+ statement.setTimestamp(paramIndex++, new java.sql.Timestamp(historyQuery.getEndDate().getTime()));
+ }
+
+ // set the user id as necessary
+ if (historyQuery.getUserName() != null) {
+ statement.setString(paramIndex++, "%" + historyQuery.getUserName().toUpperCase() + "%");
+ }
+
+ // set the source id as necessary
+ if (historyQuery.getSourceId() != null) {
+ statement.setString(paramIndex++, historyQuery.getSourceId());
+ }
+
+ // set the limit
+ statement.setInt(paramIndex++, historyQuery.getCount());
+
+ // set the offset according to the currented page calculated above
+ statement.setInt(paramIndex, historyQuery.getOffset());
+
+ // execute the query
+ rs = statement.executeQuery();
+
+ // create each corresponding action
+ while (rs.next()) {
+ final Integer actionId = rs.getInt("ID");
+ final Operation operation = Operation.valueOf(rs.getString("OPERATION"));
+ final Component component = Component.valueOf(rs.getString("SOURCE_TYPE"));
+
+ Action action = new Action();
+ action.setId(actionId);
+ action.setUserDn(rs.getString("USER_DN"));
+ action.setUserName(rs.getString("USER_NAME"));
+ action.setOperation(Operation.valueOf(rs.getString("OPERATION")));
+ action.setTimestamp(new Date(rs.getTimestamp("ACTION_TIMESTAMP").getTime()));
+ action.setSourceId(rs.getString("SOURCE_ID"));
+ action.setSourceName(rs.getString("SOURCE_NAME"));
+ action.setSourceType(Component.valueOf(rs.getString("SOURCE_TYPE")));
+
+ // get the component details if appropriate
+ ComponentDetails componentDetails = null;
+ if (Component.Processor.equals(component) || Component.ControllerService.equals(component) || Component.ReportingTask.equals(component)) {
+ componentDetails = getExtensionDetails(actionId);
+ } else if (Component.RemoteProcessGroup.equals(component)) {
+ componentDetails = getRemoteProcessGroupDetails(actionId);
+ }
+
+ if (componentDetails != null) {
+ action.setComponentDetails(componentDetails);
+ }
+
+ // get the action details if appropriate
+ ActionDetails actionDetails = null;
+ if (Operation.Move.equals(operation)) {
+ actionDetails = getMoveDetails(actionId);
+ } else if (Operation.Configure.equals(operation)) {
+ actionDetails = getConfigureDetails(actionId);
+ } else if (Operation.Connect.equals(operation) || Operation.Disconnect.equals(operation)) {
+ actionDetails = getConnectDetails(actionId);
+ } else if (Operation.Purge.equals(operation)) {
+ actionDetails = getPurgeDetails(actionId);
+ }
+
+ // set the action details
+ if (actionDetails != null) {
+ action.setActionDetails(actionDetails);
+ }
+
+ // add the action
+ actions.add(action);
+ }
+
+ // populate the action result
+ actionResult.setActions(actions);
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } finally {
+ RepositoryUtils.closeQuietly(rs);
+ RepositoryUtils.closeQuietly(statement);
+ }
+
+ return actionResult;
+ }
+
+ @Override
+ public Action getAction(Integer actionId) throws DataAccessException {
+ Action action = null;
+ PreparedStatement statement = null;
+ ResultSet rs = null;
+ try {
+ // create the statement
+ statement = connection.prepareStatement(SELECT_ACTION_BY_ID);
+ statement.setInt(1, actionId);
+
+ // execute the query
+ rs = statement.executeQuery();
+
+ // ensure results
+ if (rs.next()) {
+ Operation operation = Operation.valueOf(rs.getString("OPERATION"));
+ Component component = Component.valueOf(rs.getString("SOURCE_TYPE"));
+
+ // populate the action
+ action = new Action();
+ action.setId(rs.getInt("ID"));
+ action.setUserDn(rs.getString("USER_DN"));
+ action.setUserName(rs.getString("USER_NAME"));
+ action.setOperation(operation);
+ action.setTimestamp(new Date(rs.getTimestamp("ACTION_TIMESTAMP").getTime()));
+ action.setSourceId(rs.getString("SOURCE_ID"));
+ action.setSourceName(rs.getString("SOURCE_NAME"));
+ action.setSourceType(component);
+
+ // get the component details if appropriate
+ ComponentDetails componentDetails = null;
+ if (Component.Processor.equals(component) || Component.ControllerService.equals(component) || Component.ReportingTask.equals(component)) {
+ componentDetails = getExtensionDetails(actionId);
+ } else if (Component.RemoteProcessGroup.equals(component)) {
+ componentDetails = getRemoteProcessGroupDetails(actionId);
+ }
+
+ if (componentDetails != null) {
+ action.setComponentDetails(componentDetails);
+ }
+
+ // get the action details if appropriate
+ ActionDetails actionDetails = null;
+ if (Operation.Move.equals(operation)) {
+ actionDetails = getMoveDetails(actionId);
+ } else if (Operation.Configure.equals(operation)) {
+ actionDetails = getConfigureDetails(actionId);
+ } else if (Operation.Connect.equals(operation) || Operation.Disconnect.equals(operation)) {
+ actionDetails = getConnectDetails(actionId);
+ } else if (Operation.Purge.equals(operation)) {
+ actionDetails = getPurgeDetails(actionId);
+ }
+
+ // set the action details
+ if (actionDetails != null) {
+ action.setActionDetails(actionDetails);
+ }
+ }
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } finally {
+ RepositoryUtils.closeQuietly(rs);
+ RepositoryUtils.closeQuietly(statement);
+ }
+
+ return action;
+ }
+
+ private ExtensionDetails getExtensionDetails(Integer actionId) throws DataAccessException {
+ ExtensionDetails extensionDetails = null;
+ PreparedStatement statement = null;
+ ResultSet rs = null;
+ try {
+ // create the statement
+ statement = connection.prepareStatement(SELECT_EXTENSION_DETAILS_FOR_ACTION);
+ statement.setInt(1, actionId);
+
+ // execute the query
+ rs = statement.executeQuery();
+
+ // ensure results
+ if (rs.next()) {
+ extensionDetails = new ExtensionDetails();
+ extensionDetails.setType(rs.getString("TYPE"));
+ }
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } finally {
+ RepositoryUtils.closeQuietly(rs);
+ RepositoryUtils.closeQuietly(statement);
+ }
+
+ return extensionDetails;
+ }
+
+ private RemoteProcessGroupDetails getRemoteProcessGroupDetails(Integer actionId) throws DataAccessException {
+ RemoteProcessGroupDetails remoteProcessGroupDetails = null;
+ PreparedStatement statement = null;
+ ResultSet rs = null;
+ try {
+ // create the statement
+ statement = connection.prepareStatement(SELECT_REMOTE_PROCESS_GROUP_DETAILS_FOR_ACTION);
+ statement.setInt(1, actionId);
+
+ // execute the query
+ rs = statement.executeQuery();
+
+ // ensure results
+ if (rs.next()) {
+ remoteProcessGroupDetails = new RemoteProcessGroupDetails();
+ remoteProcessGroupDetails.setUri(rs.getString("URI"));
+ }
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } finally {
+ RepositoryUtils.closeQuietly(rs);
+ RepositoryUtils.closeQuietly(statement);
+ }
+
+ return remoteProcessGroupDetails;
+ }
+
+ private MoveDetails getMoveDetails(Integer actionId) throws DataAccessException {
+ MoveDetails moveDetails = null;
+ PreparedStatement statement = null;
+ ResultSet rs = null;
+ try {
+ // create the statement
+ statement = connection.prepareStatement(SELECT_MOVE_DETAILS_FOR_ACTION);
+ statement.setInt(1, actionId);
+
+ // execute the query
+ rs = statement.executeQuery();
+
+ // ensure results
+ if (rs.next()) {
+ moveDetails = new MoveDetails();
+ moveDetails.setGroupId(rs.getString("GROUP_ID"));
+ moveDetails.setGroup(rs.getString("GROUP_NAME"));
+ moveDetails.setPreviousGroupId(rs.getString("PREVIOUS_GROUP_ID"));
+ moveDetails.setPreviousGroup(rs.getString("PREVIOUS_GROUP_NAME"));
+ }
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } finally {
+ RepositoryUtils.closeQuietly(rs);
+ RepositoryUtils.closeQuietly(statement);
+ }
+
+ return moveDetails;
+ }
+
+ private ConnectDetails getConnectDetails(Integer actionId) throws DataAccessException {
+ ConnectDetails connectionDetails = null;
+ PreparedStatement statement = null;
+ ResultSet rs = null;
+ try {
+ // create the statement
+ statement = connection.prepareStatement(SELECT_CONNECT_DETAILS_FOR_ACTION);
+ statement.setInt(1, actionId);
+
+ // execute the query
+ rs = statement.executeQuery();
+
+ // ensure results
+ if (rs.next()) {
+ final Component sourceComponent = Component.valueOf(rs.getString("SOURCE_TYPE"));
+ final Component destinationComponent = Component.valueOf(rs.getString("DESTINATION_TYPE"));
+
+ connectionDetails = new ConnectDetails();
+ connectionDetails.setSourceId(rs.getString("SOURCE_ID"));
+ connectionDetails.setSourceName(rs.getString("SOURCE_NAME"));
+ connectionDetails.setSourceType(sourceComponent);
+ connectionDetails.setRelationship(rs.getString("RELATIONSHIP"));
+ connectionDetails.setDestinationId(rs.getString("DESTINATION_ID"));
+ connectionDetails.setDestinationName(rs.getString("DESTINATION_NAME"));
+ connectionDetails.setDestinationType(destinationComponent);
+ }
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } finally {
+ RepositoryUtils.closeQuietly(rs);
+ RepositoryUtils.closeQuietly(statement);
+ }
+
+ return connectionDetails;
+ }
+
+ private ConfigureDetails getConfigureDetails(Integer actionId) throws DataAccessException {
+ ConfigureDetails configurationDetails = null;
+ PreparedStatement statement = null;
+ ResultSet rs = null;
+ try {
+ // create the statement
+ statement = connection.prepareStatement(SELECT_CONFIGURE_DETAILS_FOR_ACTION);
+ statement.setInt(1, actionId);
+
+ // execute the query
+ rs = statement.executeQuery();
+
+ // ensure results
+ if (rs.next()) {
+ configurationDetails = new ConfigureDetails();
+ configurationDetails.setName(rs.getString("NAME"));
+ configurationDetails.setValue(rs.getString("VALUE"));
+ configurationDetails.setPreviousValue(rs.getString("PREVIOUS_VALUE"));
+ }
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } finally {
+ RepositoryUtils.closeQuietly(rs);
+ RepositoryUtils.closeQuietly(statement);
+ }
+
+ return configurationDetails;
+ }
+
+ private PurgeDetails getPurgeDetails(Integer actionId) throws DataAccessException {
+ PurgeDetails purgeDetails = null;
+ PreparedStatement statement = null;
+ ResultSet rs = null;
+ try {
+ // create the statement
+ statement = connection.prepareStatement(SELECT_PURGE_DETAILS_FOR_ACTION);
+ statement.setInt(1, actionId);
+
+ // execute the query
+ rs = statement.executeQuery();
+
+ // ensure results
+ if (rs.next()) {
+ purgeDetails = new PurgeDetails();
+ purgeDetails.setEndDate(new Date(rs.getTimestamp("END_DATE").getTime()));
+ }
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } finally {
+ RepositoryUtils.closeQuietly(rs);
+ RepositoryUtils.closeQuietly(statement);
+ }
+
+ return purgeDetails;
+ }
+
+ @Override
+ public Map<String, List<PreviousValue>> getPreviousValues(String componentId) {
+ Map<String, List<PreviousValue>> previousValues = new LinkedHashMap<>();
+
+ PreparedStatement statement = null;
+ ResultSet rs = null;
+ try {
+ // create the statement
+ statement = connection.prepareStatement(SELECT_PREVIOUSLY_CONFIGURED_FIELDS);
+ statement.setString(1, componentId);
+
+ // execute the query
+ rs = statement.executeQuery();
+
+ // ensure results
+ while (rs.next()) {
+ final String property = rs.getString("NAME");
+ previousValues.put(property, getPreviousValuesForProperty(componentId, property));
+ }
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } finally {
+ RepositoryUtils.closeQuietly(rs);
+ RepositoryUtils.closeQuietly(statement);
+ }
+
+ return previousValues;
+ }
+
+ private List<PreviousValue> getPreviousValuesForProperty(final String componentId, final String property) {
+ List<PreviousValue> previousValues = new ArrayList<>();
+
+ PreparedStatement statement = null;
+ ResultSet rs = null;
+ try {
+ // create the statement
+ statement = connection.prepareStatement(SELECT_PREVIOUS_VALUES);
+ statement.setString(1, componentId);
+ statement.setString(2, property);
+
+ // execute the query
+ rs = statement.executeQuery();
+
+ // ensure results
+ while (rs.next()) {
+ // get the previous value
+ final PreviousValue previousValue = new PreviousValue();
+ previousValue.setPreviousValue(rs.getString("VALUE"));
+ previousValue.setTimestamp(new Date(rs.getTimestamp("ACTION_TIMESTAMP").getTime()));
+ previousValue.setUserName(rs.getString("USER_NAME"));
+ previousValues.add(previousValue);
+ }
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } finally {
+ RepositoryUtils.closeQuietly(rs);
+ RepositoryUtils.closeQuietly(statement);
+ }
+
+ return previousValues;
+ }
+
+ @Override
+ public void deleteActions(Date endDate) throws DataAccessException {
+ PreparedStatement statement = null;
+ try {
+ // -----------------
+ // component details
+ // -----------------
+
+ // create the move delete statement
+ statement = connection.prepareStatement(String.format(DELETE_SPECIFIC_ACTIONS, "PROCESSOR_DETAILS", "ACTION_ID"));
+ statement.setTimestamp(1, new java.sql.Timestamp(endDate.getTime()));
+ statement.executeUpdate();
+ statement.close();
+
+ // create the move delete statement
+ statement = connection.prepareStatement(String.format(DELETE_SPECIFIC_ACTIONS, "REMOTE_PROCESS_GROUP_DETAILS", "ACTION_ID"));
+ statement.setTimestamp(1, new java.sql.Timestamp(endDate.getTime()));
+ statement.executeUpdate();
+ statement.close();
+
+ // --------------
+ // action details
+ // --------------
+ // create the move delete statement
+ statement = connection.prepareStatement(String.format(DELETE_SPECIFIC_ACTIONS, "MOVE_DETAILS", "ACTION_ID"));
+ statement.setTimestamp(1, new java.sql.Timestamp(endDate.getTime()));
+ statement.executeUpdate();
+ statement.close();
+
+ // create the configure delete statement
+ statement = connection.prepareStatement(String.format(DELETE_SPECIFIC_ACTIONS, "CONFIGURE_DETAILS", "ACTION_ID"));
+ statement.setTimestamp(1, new java.sql.Timestamp(endDate.getTime()));
+ statement.executeUpdate();
+ statement.close();
+
+ // create the connect delete statement
+ statement = connection.prepareStatement(String.format(DELETE_SPECIFIC_ACTIONS, "CONNECT_DETAILS", "ACTION_ID"));
+ statement.setTimestamp(1, new java.sql.Timestamp(endDate.getTime()));
+ statement.executeUpdate();
+ statement.close();
+
+ // create the relationship delete statement
+ statement = connection.prepareStatement(String.format(DELETE_SPECIFIC_ACTIONS, "PURGE_DETAILS", "ACTION_ID"));
+ statement.setTimestamp(1, new java.sql.Timestamp(endDate.getTime()));
+ statement.executeUpdate();
+ statement.close();
+
+ // -------
+ // actions
+ // -------
+ // create the action delete statement
+ statement = connection.prepareStatement(DELETE_ACTIONS);
+ statement.setTimestamp(1, new java.sql.Timestamp(endDate.getTime()));
+ statement.executeUpdate();
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } finally {
+ RepositoryUtils.closeQuietly(statement);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/impl/StandardAuthorityDAO.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/impl/StandardAuthorityDAO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/impl/StandardAuthorityDAO.java
new file mode 100644
index 0000000..4e2cc26
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/impl/StandardAuthorityDAO.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.dao.impl;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.EnumSet;
+import java.util.Set;
+import org.apache.nifi.admin.RepositoryUtils;
+import org.apache.nifi.admin.dao.AuthorityDAO;
+import org.apache.nifi.admin.dao.DataAccessException;
+import org.apache.nifi.authorization.Authority;
+
+/**
+ *
+ */
+public class StandardAuthorityDAO implements AuthorityDAO {
+
+ private static final String SELECT_AUTHORITIES_FOR_USER = "SELECT ID, ROLE "
+ + "FROM AUTHORITY "
+ + "WHERE USER_ID = ?";
+
+ private static final String INSERT_AUTHORITY = "INSERT INTO AUTHORITY ("
+ + "USER_ID, ROLE"
+ + ") VALUES ("
+ + "?, ?"
+ + ")";
+
+ private static final String DELETE_AUTHORITY = "DELETE FROM AUTHORITY "
+ + "WHERE USER_ID = ? AND ROLE = ?";
+
+ private static final String DELETE_AUTHORITIES_FOR_USER = "DELETE FROM AUTHORITY "
+ + "WHERE USER_ID = ?";
+
+ private final Connection connection;
+
+ public StandardAuthorityDAO(Connection connection) {
+ this.connection = connection;
+ }
+
+ @Override
+ public void createAuthorities(Set<Authority> authorities, String userId) throws DataAccessException {
+ if (authorities == null) {
+ throw new IllegalArgumentException("Specified authorities cannot be null.");
+ }
+
+ // ensure there are some authorities to create
+ if (!authorities.isEmpty()) {
+ PreparedStatement statement = null;
+ try {
+ // add each authority for the specified user
+ statement = connection.prepareStatement(INSERT_AUTHORITY);
+ statement.setString(1, userId);
+ for (Authority authority : authorities) {
+ statement.setString(2, authority.toString());
+ statement.addBatch();
+ }
+
+ // insert the authorities
+ int[] updateCounts = statement.executeBatch();
+ for (int updateCount : updateCounts) {
+ if (updateCount != 1) {
+ throw new DataAccessException("Unable to insert user authorities.");
+ }
+ }
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } catch (DataAccessException dae) {
+ throw dae;
+ } finally {
+ RepositoryUtils.closeQuietly(statement);
+ }
+ }
+ }
+
+ @Override
+ public void deleteAuthorities(String userId) throws DataAccessException {
+ // ensure there are some authorities to create
+ PreparedStatement statement = null;
+ try {
+ // add each authority for the specified user
+ statement = connection.prepareStatement(DELETE_AUTHORITIES_FOR_USER);
+ statement.setString(1, userId);
+
+ // insert the authorities
+ statement.executeUpdate();
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } finally {
+ RepositoryUtils.closeQuietly(statement);
+ }
+ }
+
+ @Override
+ public void deleteAuthorities(Set<Authority> authorities, String userId) throws DataAccessException {
+ if (authorities == null) {
+ throw new IllegalArgumentException("Specified authorities cannot be null.");
+ }
+
+ // ensure there are some authorities to create
+ if (!authorities.isEmpty()) {
+ PreparedStatement statement = null;
+ try {
+ // add each authority for the specified user
+ statement = connection.prepareStatement(DELETE_AUTHORITY);
+ statement.setString(1, userId);
+ for (Authority authority : authorities) {
+ statement.setString(2, authority.toString());
+ statement.addBatch();
+ }
+
+ // insert the authorities
+ int[] updateCounts = statement.executeBatch();
+ for (int updateCount : updateCounts) {
+ if (updateCount != 1) {
+ throw new DataAccessException("Unable to remove user authorities.");
+ }
+ }
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } catch (DataAccessException dae) {
+ throw dae;
+ } finally {
+ RepositoryUtils.closeQuietly(statement);
+ }
+ }
+ }
+
+ @Override
+ public Set<Authority> findAuthoritiesByUserId(String userId) throws DataAccessException {
+ Set<Authority> authorities = EnumSet.noneOf(Authority.class);
+ PreparedStatement statement = null;
+ ResultSet rs = null;
+ try {
+ // add each authority for the specified user
+ statement = connection.prepareStatement(SELECT_AUTHORITIES_FOR_USER);
+ statement.setString(1, userId);
+
+ // execute the query
+ rs = statement.executeQuery();
+
+ // create each corresponding authority
+ while (rs.next()) {
+ authorities.add(Authority.valueOfAuthority(rs.getString("ROLE")));
+ }
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } finally {
+ RepositoryUtils.closeQuietly(rs);
+ RepositoryUtils.closeQuietly(statement);
+ }
+
+ return authorities;
+ }
+
+}
[28/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/client/socket/EndpointConnectionPool.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/client/socket/EndpointConnectionPool.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/client/socket/EndpointConnectionPool.java
new file mode 100644
index 0000000..b162b3f
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/client/socket/EndpointConnectionPool.java
@@ -0,0 +1,994 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.client.socket;
+
+import java.io.BufferedReader;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.channels.SocketChannel;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.regex.Pattern;
+
+import javax.net.ssl.SSLContext;
+import javax.security.cert.CertificateExpiredException;
+import javax.security.cert.CertificateNotYetValidException;
+
+import org.apache.nifi.events.EventReporter;
+import org.apache.nifi.remote.Peer;
+import org.apache.nifi.remote.PeerDescription;
+import org.apache.nifi.remote.PeerStatus;
+import org.apache.nifi.remote.RemoteDestination;
+import org.apache.nifi.remote.RemoteResourceInitiator;
+import org.apache.nifi.remote.TransferDirection;
+import org.apache.nifi.remote.client.SiteToSiteClientConfig;
+import org.apache.nifi.remote.cluster.ClusterNodeInformation;
+import org.apache.nifi.remote.cluster.NodeInformation;
+import org.apache.nifi.remote.codec.FlowFileCodec;
+import org.apache.nifi.remote.exception.HandshakeException;
+import org.apache.nifi.remote.exception.PortNotRunningException;
+import org.apache.nifi.remote.exception.ProtocolException;
+import org.apache.nifi.remote.exception.TransmissionDisabledException;
+import org.apache.nifi.remote.exception.UnknownPortException;
+import org.apache.nifi.remote.io.socket.SocketChannelCommunicationsSession;
+import org.apache.nifi.remote.io.socket.ssl.SSLSocketChannel;
+import org.apache.nifi.remote.io.socket.ssl.SSLSocketChannelCommunicationsSession;
+import org.apache.nifi.remote.protocol.CommunicationsSession;
+import org.apache.nifi.remote.protocol.socket.SocketClientProtocol;
+import org.apache.nifi.remote.util.NiFiRestApiUtil;
+import org.apache.nifi.remote.util.PeerStatusCache;
+import org.apache.nifi.reporting.Severity;
+import org.apache.nifi.stream.io.BufferedOutputStream;
+import org.apache.nifi.web.api.dto.ControllerDTO;
+import org.apache.nifi.web.api.dto.PortDTO;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.helpers.MessageFormatter;
+
+public class EndpointConnectionPool {
+
+ public static final long PEER_REFRESH_PERIOD = 60000L;
+ public static final String CATEGORY = "Site-to-Site";
+ public static final long REMOTE_REFRESH_MILLIS = TimeUnit.MILLISECONDS.convert(10, TimeUnit.MINUTES);
+
+ private static final long PEER_CACHE_MILLIS = TimeUnit.MILLISECONDS.convert(1, TimeUnit.MINUTES);
+
+ private static final Logger logger = LoggerFactory.getLogger(EndpointConnectionPool.class);
+
+ private final ConcurrentMap<PeerDescription, BlockingQueue<EndpointConnection>> connectionQueueMap = new ConcurrentHashMap<>();
+ private final ConcurrentMap<PeerDescription, Long> peerTimeoutExpirations = new ConcurrentHashMap<>();
+ private final URI clusterUrl;
+ private final String apiUri;
+
+ private final AtomicLong peerIndex = new AtomicLong(0L);
+
+ private final ReentrantLock peerRefreshLock = new ReentrantLock();
+ private volatile List<PeerStatus> peerStatuses;
+ private volatile long peerRefreshTime = 0L;
+ private volatile PeerStatusCache peerStatusCache;
+ private final Set<EndpointConnection> activeConnections = Collections.synchronizedSet(new HashSet<EndpointConnection>());
+
+ private final File peersFile;
+ private final EventReporter eventReporter;
+ private final SSLContext sslContext;
+ private final ScheduledExecutorService taskExecutor;
+ private final int idleExpirationMillis;
+ private final RemoteDestination remoteDestination;
+
+ private final ReadWriteLock listeningPortRWLock = new ReentrantReadWriteLock();
+ private final Lock remoteInfoReadLock = listeningPortRWLock.readLock();
+ private final Lock remoteInfoWriteLock = listeningPortRWLock.writeLock();
+ private Integer siteToSitePort;
+ private Boolean siteToSiteSecure;
+ private long remoteRefreshTime;
+ private final Map<String, String> inputPortMap = new HashMap<>(); // map input port name to identifier
+ private final Map<String, String> outputPortMap = new HashMap<>(); // map output port name to identifier
+
+ private volatile int commsTimeout;
+ private volatile boolean shutdown = false;
+
+ public EndpointConnectionPool(final String clusterUrl, final RemoteDestination remoteDestination, final int commsTimeoutMillis,
+ final int idleExpirationMillis, final EventReporter eventReporter, final File persistenceFile) {
+ this(clusterUrl, remoteDestination, commsTimeoutMillis, idleExpirationMillis, null, eventReporter, persistenceFile);
+ }
+
+ public EndpointConnectionPool(final String clusterUrl, final RemoteDestination remoteDestination, final int commsTimeoutMillis, final int idleExpirationMillis,
+ final SSLContext sslContext, final EventReporter eventReporter, final File persistenceFile) {
+ Objects.requireNonNull(clusterUrl, "URL cannot be null");
+ Objects.requireNonNull(remoteDestination, "Remote Destination/Port Identifier cannot be null");
+ try {
+ this.clusterUrl = new URI(clusterUrl);
+ } catch (final URISyntaxException e) {
+ throw new IllegalArgumentException("Invalid Cluster URL: " + clusterUrl);
+ }
+
+ // Trim the trailing /
+ String uriPath = this.clusterUrl.getPath();
+ if (uriPath.endsWith("/")) {
+ uriPath = uriPath.substring(0, uriPath.length() - 1);
+ }
+ apiUri = this.clusterUrl.getScheme() + "://" + this.clusterUrl.getHost() + ":" + this.clusterUrl.getPort() + uriPath + "-api";
+
+ this.remoteDestination = remoteDestination;
+ this.sslContext = sslContext;
+ this.peersFile = persistenceFile;
+ this.eventReporter = eventReporter;
+ this.commsTimeout = commsTimeoutMillis;
+ this.idleExpirationMillis = idleExpirationMillis;
+
+ Set<PeerStatus> recoveredStatuses;
+ if (persistenceFile != null && persistenceFile.exists()) {
+ try {
+ recoveredStatuses = recoverPersistedPeerStatuses(peersFile);
+ this.peerStatusCache = new PeerStatusCache(recoveredStatuses, peersFile.lastModified());
+ } catch (final IOException ioe) {
+ logger.warn("Failed to recover peer statuses from {} due to {}; will continue without loading information from file", persistenceFile, ioe);
+ }
+ } else {
+ peerStatusCache = null;
+ }
+
+ // Initialize a scheduled executor and run some maintenance tasks in the background to kill off old, unused
+ // connections and keep our list of peers up-to-date.
+ taskExecutor = Executors.newScheduledThreadPool(1, new ThreadFactory() {
+ private final ThreadFactory defaultFactory = Executors.defaultThreadFactory();
+
+ @Override
+ public Thread newThread(final Runnable r) {
+ final Thread thread = defaultFactory.newThread(r);
+ thread.setName("NiFi Site-to-Site Connection Pool Maintenance");
+ return thread;
+ }
+ });
+
+ taskExecutor.scheduleWithFixedDelay(new Runnable() {
+ @Override
+ public void run() {
+ refreshPeers();
+ }
+ }, 0, 5, TimeUnit.SECONDS);
+
+ taskExecutor.scheduleWithFixedDelay(new Runnable() {
+ @Override
+ public void run() {
+ cleanupExpiredSockets();
+ }
+ }, 5, 5, TimeUnit.SECONDS);
+ }
+
+ void warn(final String msg, final Object... args) {
+ logger.warn(msg, args);
+ if (eventReporter != null) {
+ eventReporter.reportEvent(Severity.WARNING, "Site-to-Site", MessageFormatter.arrayFormat(msg, args).getMessage());
+ }
+ }
+
+ void warn(final String msg, final Throwable t) {
+ logger.warn(msg, t);
+
+ if (eventReporter != null) {
+ eventReporter.reportEvent(Severity.WARNING, "Site-to-Site", msg + ": " + t.toString());
+ }
+ }
+
+ void error(final String msg, final Object... args) {
+ logger.error(msg, args);
+ if (eventReporter != null) {
+ eventReporter.reportEvent(Severity.ERROR, "Site-to-Site", MessageFormatter.arrayFormat(msg, args).getMessage());
+ }
+ }
+
+ private String getPortIdentifier(final TransferDirection transferDirection) throws IOException {
+ if (remoteDestination.getIdentifier() != null) {
+ return remoteDestination.getIdentifier();
+ }
+
+ if (transferDirection == TransferDirection.RECEIVE) {
+ return getOutputPortIdentifier(remoteDestination.getName());
+ } else {
+ return getInputPortIdentifier(remoteDestination.getName());
+ }
+ }
+
+ public EndpointConnection getEndpointConnection(final TransferDirection direction) throws IOException, HandshakeException, PortNotRunningException, UnknownPortException, ProtocolException {
+ return getEndpointConnection(direction, null);
+ }
+
+ public EndpointConnection getEndpointConnection(final TransferDirection direction, final SiteToSiteClientConfig config)
+ throws IOException, HandshakeException, PortNotRunningException, UnknownPortException, ProtocolException {
+ //
+ // Attempt to get a connection state that already exists for this URL.
+ //
+ FlowFileCodec codec = null;
+ CommunicationsSession commsSession = null;
+ SocketClientProtocol protocol = null;
+ EndpointConnection connection;
+ Peer peer = null;
+
+ logger.debug("{} getting next peer status", this);
+ final PeerStatus peerStatus = getNextPeerStatus(direction);
+ logger.debug("{} next peer status = {}", this, peerStatus);
+ if (peerStatus == null) {
+ return null;
+ }
+
+ final PeerDescription peerDescription = peerStatus.getPeerDescription();
+ BlockingQueue<EndpointConnection> connectionQueue = connectionQueueMap.get(peerDescription);
+ if (connectionQueue == null) {
+ connectionQueue = new LinkedBlockingQueue<>();
+ BlockingQueue<EndpointConnection> existing = connectionQueueMap.putIfAbsent(peerDescription, connectionQueue);
+ if (existing != null) {
+ connectionQueue = existing;
+ }
+ }
+
+ final List<EndpointConnection> addBack = new ArrayList<>();
+ try {
+ do {
+ connection = connectionQueue.poll();
+ logger.debug("{} Connection State for {} = {}", this, clusterUrl, connection);
+ final String portId = getPortIdentifier(direction);
+
+ if (connection == null && !addBack.isEmpty()) {
+ // all available connections have been penalized.
+ logger.debug("{} all Connections for {} are penalized; returning no Connection", this, portId);
+ return null;
+ }
+
+ if (connection != null && connection.getPeer().isPenalized(portId)) {
+ // we have a connection, but it's penalized. We want to add it back to the queue
+ // when we've found one to use.
+ addBack.add(connection);
+ continue;
+ }
+
+ // if we can't get an existing Connection, create one
+ if (connection == null) {
+ logger.debug("{} No Connection available for Port {}; creating new Connection", this, portId);
+ protocol = new SocketClientProtocol();
+ protocol.setDestination(new IdEnrichedRemoteDestination(remoteDestination, portId));
+ protocol.setEventReporter(eventReporter);
+
+ final long penalizationMillis = remoteDestination.getYieldPeriod(TimeUnit.MILLISECONDS);
+ try {
+ logger.debug("{} Establishing site-to-site connection with {}", this, peerStatus);
+ commsSession = establishSiteToSiteConnection(peerStatus);
+ } catch (final IOException ioe) {
+ penalize(peerStatus.getPeerDescription(), penalizationMillis);
+ throw ioe;
+ }
+
+ final DataInputStream dis = new DataInputStream(commsSession.getInput().getInputStream());
+ final DataOutputStream dos = new DataOutputStream(commsSession.getOutput().getOutputStream());
+ try {
+ logger.debug("{} Negotiating protocol", this);
+ RemoteResourceInitiator.initiateResourceNegotiation(protocol, dis, dos);
+ } catch (final HandshakeException e) {
+ try {
+ commsSession.close();
+ } catch (final IOException ioe) {
+ throw e;
+ }
+ }
+
+ final String peerUrl = "nifi://" + peerDescription.getHostname() + ":" + peerDescription.getPort();
+ peer = new Peer(peerDescription, commsSession, peerUrl, clusterUrl.toString());
+
+ // set properties based on config
+ if (config != null) {
+ protocol.setTimeout((int) config.getTimeout(TimeUnit.MILLISECONDS));
+ protocol.setPreferredBatchCount(config.getPreferredBatchCount());
+ protocol.setPreferredBatchSize(config.getPreferredBatchSize());
+ protocol.setPreferredBatchDuration(config.getPreferredBatchDuration(TimeUnit.MILLISECONDS));
+ }
+
+ // perform handshake
+ try {
+ logger.debug("{} performing handshake", this);
+ protocol.handshake(peer);
+
+ // handle error cases
+ if (protocol.isDestinationFull()) {
+ logger.warn("{} {} indicates that port {}'s destination is full; penalizing peer",
+ this, peer, config.getPortName() == null ? config.getPortIdentifier() : config.getPortName());
+
+ penalize(peer, penalizationMillis);
+ try {
+ peer.close();
+ } catch (final IOException ioe) {
+ }
+
+ continue;
+ } else if (protocol.isPortInvalid()) {
+ penalize(peer, penalizationMillis);
+ cleanup(protocol, peer);
+ throw new PortNotRunningException(peer.toString() + " indicates that port " + portId + " is not running");
+ } else if (protocol.isPortUnknown()) {
+ penalize(peer, penalizationMillis);
+ cleanup(protocol, peer);
+ throw new UnknownPortException(peer.toString() + " indicates that port " + portId + " is not known");
+ }
+
+ // negotiate the FlowFileCodec to use
+ logger.debug("{} negotiating codec", this);
+ codec = protocol.negotiateCodec(peer);
+ logger.debug("{} negotiated codec is {}", this, codec);
+ } catch (final PortNotRunningException | UnknownPortException e) {
+ throw e;
+ } catch (final Exception e) {
+ penalize(peer, penalizationMillis);
+ cleanup(protocol, peer);
+
+ final String message = String.format("%s failed to communicate with %s due to %s", this, peer == null ? clusterUrl : peer, e.toString());
+ error(message);
+ if (logger.isDebugEnabled()) {
+ logger.error("", e);
+ }
+ throw e;
+ }
+
+ connection = new EndpointConnection(peer, protocol, codec);
+ } else {
+ final long lastTimeUsed = connection.getLastTimeUsed();
+ final long millisSinceLastUse = System.currentTimeMillis() - lastTimeUsed;
+
+ if (commsTimeout > 0L && millisSinceLastUse >= commsTimeout) {
+ cleanup(connection.getSocketClientProtocol(), connection.getPeer());
+ connection = null;
+ } else {
+ codec = connection.getCodec();
+ peer = connection.getPeer();
+ commsSession = peer.getCommunicationsSession();
+ protocol = connection.getSocketClientProtocol();
+ }
+ }
+ } while (connection == null || codec == null || commsSession == null || protocol == null);
+ } catch (final Throwable t) {
+ if (commsSession != null) {
+ try {
+ commsSession.close();
+ } catch (final IOException ioe) {
+ }
+ }
+
+ throw t;
+ } finally {
+ if (!addBack.isEmpty()) {
+ connectionQueue.addAll(addBack);
+ }
+ }
+
+ activeConnections.add(connection);
+ return connection;
+ }
+
+ public boolean offer(final EndpointConnection endpointConnection) {
+ final Peer peer = endpointConnection.getPeer();
+ if (peer == null) {
+ return false;
+ }
+
+ final BlockingQueue<EndpointConnection> connectionQueue = connectionQueueMap.get(peer.getDescription());
+ if (connectionQueue == null) {
+ return false;
+ }
+
+ activeConnections.remove(endpointConnection);
+ if (shutdown) {
+ terminate(endpointConnection);
+ return false;
+ } else {
+ endpointConnection.setLastTimeUsed();
+ return connectionQueue.offer(endpointConnection);
+ }
+ }
+
+ private void penalize(final PeerDescription peerDescription, final long penalizationMillis) {
+ Long expiration = peerTimeoutExpirations.get(peerDescription);
+ if (expiration == null) {
+ expiration = Long.valueOf(0L);
+ }
+
+ final long newExpiration = Math.max(expiration, System.currentTimeMillis() + penalizationMillis);
+ peerTimeoutExpirations.put(peerDescription, Long.valueOf(newExpiration));
+ }
+
+ /**
+ * Updates internal state map to penalize a PeerStatus that points to the
+ * specified peer
+ *
+ * @param peer the peer
+ * @param penalizationMillis period of time to penalize a given peer
+ */
+ public void penalize(final Peer peer, final long penalizationMillis) {
+ penalize(peer.getDescription(), penalizationMillis);
+ }
+
+ private void cleanup(final SocketClientProtocol protocol, final Peer peer) {
+ if (protocol != null && peer != null) {
+ try {
+ protocol.shutdown(peer);
+ } catch (final TransmissionDisabledException e) {
+ // User disabled transmission.... do nothing.
+ logger.debug(this + " Transmission Disabled by User");
+ } catch (IOException e1) {
+ }
+ }
+
+ if (peer != null) {
+ try {
+ peer.close();
+ } catch (final TransmissionDisabledException e) {
+ // User disabled transmission.... do nothing.
+ logger.debug(this + " Transmission Disabled by User");
+ } catch (IOException e1) {
+ }
+ }
+ }
+
+ private boolean isPeerRefreshNeeded(final List<PeerStatus> peerList) {
+ return (peerList == null || peerList.isEmpty() || System.currentTimeMillis() > peerRefreshTime + PEER_REFRESH_PERIOD);
+ }
+
+ private PeerStatus getNextPeerStatus(final TransferDirection direction) {
+ List<PeerStatus> peerList = peerStatuses;
+ if (isPeerRefreshNeeded(peerList)) {
+ peerRefreshLock.lock();
+ try {
+ // now that we have the lock, check again that we need to refresh (because another thread
+ // could have been refreshing while we were waiting for the lock).
+ peerList = peerStatuses;
+ if (isPeerRefreshNeeded(peerList)) {
+ try {
+ peerList = createPeerStatusList(direction);
+ } catch (final Exception e) {
+ final String message = String.format("%s Failed to update list of peers due to %s", this, e.toString());
+ warn(message);
+ if (logger.isDebugEnabled()) {
+ logger.warn("", e);
+ }
+
+ if (eventReporter != null) {
+ eventReporter.reportEvent(Severity.WARNING, CATEGORY, message);
+ }
+ }
+
+ this.peerStatuses = peerList;
+ peerRefreshTime = System.currentTimeMillis();
+ }
+ } finally {
+ peerRefreshLock.unlock();
+ }
+ }
+
+ if (peerList == null || peerList.isEmpty()) {
+ return null;
+ }
+
+ PeerStatus peerStatus;
+ for (int i = 0; i < peerList.size(); i++) {
+ final long idx = peerIndex.getAndIncrement();
+ final int listIndex = (int) (idx % peerList.size());
+ peerStatus = peerList.get(listIndex);
+
+ if (isPenalized(peerStatus)) {
+ logger.debug("{} {} is penalized; will not communicate with this peer", this, peerStatus);
+ } else {
+ return peerStatus;
+ }
+ }
+
+ logger.debug("{} All peers appear to be penalized; returning null", this);
+ return null;
+ }
+
+ private boolean isPenalized(final PeerStatus peerStatus) {
+ final Long expirationEnd = peerTimeoutExpirations.get(peerStatus.getPeerDescription());
+ return (expirationEnd == null ? false : expirationEnd > System.currentTimeMillis());
+ }
+
+ private List<PeerStatus> createPeerStatusList(final TransferDirection direction) throws IOException, HandshakeException, UnknownPortException, PortNotRunningException {
+ Set<PeerStatus> statuses = getPeerStatuses();
+ if (statuses == null) {
+ refreshPeers();
+ statuses = getPeerStatuses();
+ if (statuses == null) {
+ logger.debug("{} found no peers to connect to", this);
+ return Collections.emptyList();
+ }
+ }
+
+ final ClusterNodeInformation clusterNodeInfo = new ClusterNodeInformation();
+ final List<NodeInformation> nodeInfos = new ArrayList<>();
+ for (final PeerStatus peerStatus : statuses) {
+ final PeerDescription description = peerStatus.getPeerDescription();
+ final NodeInformation nodeInfo = new NodeInformation(description.getHostname(), description.getPort(), 0, description.isSecure(), peerStatus.getFlowFileCount());
+ nodeInfos.add(nodeInfo);
+ }
+ clusterNodeInfo.setNodeInformation(nodeInfos);
+ return formulateDestinationList(clusterNodeInfo, direction);
+ }
+
+ private Set<PeerStatus> getPeerStatuses() {
+ final PeerStatusCache cache = this.peerStatusCache;
+ if (cache == null || cache.getStatuses() == null || cache.getStatuses().isEmpty()) {
+ return null;
+ }
+
+ if (cache.getTimestamp() + PEER_CACHE_MILLIS < System.currentTimeMillis()) {
+ final Set<PeerStatus> equalizedSet = new HashSet<>(cache.getStatuses().size());
+ for (final PeerStatus status : cache.getStatuses()) {
+ final PeerStatus equalizedStatus = new PeerStatus(status.getPeerDescription(), 1);
+ equalizedSet.add(equalizedStatus);
+ }
+
+ return equalizedSet;
+ }
+
+ return cache.getStatuses();
+ }
+
+ private Set<PeerStatus> fetchRemotePeerStatuses() throws IOException, HandshakeException, UnknownPortException, PortNotRunningException {
+ final String hostname = clusterUrl.getHost();
+ final Integer port = getSiteToSitePort();
+ if (port == null) {
+ throw new IOException("Remote instance of NiFi is not configured to allow site-to-site communications");
+ }
+
+ final PeerDescription clusterPeerDescription = new PeerDescription(hostname, port, clusterUrl.toString().startsWith("https://"));
+ final CommunicationsSession commsSession = establishSiteToSiteConnection(hostname, port);
+ final Peer peer = new Peer(clusterPeerDescription, commsSession, "nifi://" + hostname + ":" + port, clusterUrl.toString());
+ final SocketClientProtocol clientProtocol = new SocketClientProtocol();
+ final DataInputStream dis = new DataInputStream(commsSession.getInput().getInputStream());
+ final DataOutputStream dos = new DataOutputStream(commsSession.getOutput().getOutputStream());
+ RemoteResourceInitiator.initiateResourceNegotiation(clientProtocol, dis, dos);
+
+ clientProtocol.setTimeout(commsTimeout);
+ if (clientProtocol.getVersionNegotiator().getVersion() < 5) {
+ String portId = getPortIdentifier(TransferDirection.RECEIVE);
+ if (portId == null) {
+ portId = getPortIdentifier(TransferDirection.SEND);
+ }
+
+ if (portId == null) {
+ peer.close();
+ throw new IOException("Failed to determine the identifier of port " + remoteDestination.getName());
+ }
+ clientProtocol.handshake(peer, portId);
+ } else {
+ clientProtocol.handshake(peer, null);
+ }
+
+ final Set<PeerStatus> peerStatuses = clientProtocol.getPeerStatuses(peer);
+ persistPeerStatuses(peerStatuses);
+
+ try {
+ clientProtocol.shutdown(peer);
+ } catch (final IOException e) {
+ final String message = String.format("%s Failed to shutdown protocol when updating list of peers due to %s", this, e.toString());
+ warn(message);
+ if (logger.isDebugEnabled()) {
+ logger.warn("", e);
+ }
+ }
+
+ try {
+ peer.close();
+ } catch (final IOException e) {
+ final String message = String.format("%s Failed to close resources when updating list of peers due to %s", this, e.toString());
+ warn(message);
+ if (logger.isDebugEnabled()) {
+ logger.warn("", e);
+ }
+ }
+
+ return peerStatuses;
+ }
+
+ private void persistPeerStatuses(final Set<PeerStatus> statuses) {
+ if (peersFile == null) {
+ return;
+ }
+
+ try (final OutputStream fos = new FileOutputStream(peersFile);
+ final OutputStream out = new BufferedOutputStream(fos)) {
+
+ for (final PeerStatus status : statuses) {
+ final PeerDescription description = status.getPeerDescription();
+ final String line = description.getHostname() + ":" + description.getPort() + ":" + description.isSecure() + "\n";
+ out.write(line.getBytes(StandardCharsets.UTF_8));
+ }
+
+ } catch (final IOException e) {
+ error("Failed to persist list of Peers due to {}; if restarted and peer's NCM is down, may be unable to transfer data until communications with NCM are restored", e.toString());
+ logger.error("", e);
+ }
+ }
+
+ private Set<PeerStatus> recoverPersistedPeerStatuses(final File file) throws IOException {
+ if (!file.exists()) {
+ return null;
+ }
+
+ final Set<PeerStatus> statuses = new HashSet<>();
+ try (final InputStream fis = new FileInputStream(file);
+ final BufferedReader reader = new BufferedReader(new InputStreamReader(fis))) {
+
+ String line;
+ while ((line = reader.readLine()) != null) {
+ final String[] splits = line.split(Pattern.quote(":"));
+ if (splits.length != 3) {
+ continue;
+ }
+
+ final String hostname = splits[0];
+ final int port = Integer.parseInt(splits[1]);
+ final boolean secure = Boolean.parseBoolean(splits[2]);
+
+ statuses.add(new PeerStatus(new PeerDescription(hostname, port, secure), 1));
+ }
+ }
+
+ return statuses;
+ }
+
+ private CommunicationsSession establishSiteToSiteConnection(final PeerStatus peerStatus) throws IOException {
+ final PeerDescription description = peerStatus.getPeerDescription();
+ return establishSiteToSiteConnection(description.getHostname(), description.getPort());
+ }
+
+ private CommunicationsSession establishSiteToSiteConnection(final String hostname, final int port) throws IOException {
+ final boolean siteToSiteSecure = isSecure();
+ final String destinationUri = "nifi://" + hostname + ":" + port;
+
+ CommunicationsSession commsSession = null;
+ try {
+ if (siteToSiteSecure) {
+ if (sslContext == null) {
+ throw new IOException("Unable to communicate with " + hostname + ":" + port
+ + " because it requires Secure Site-to-Site communications, but this instance is not configured for secure communications");
+ }
+
+ final SSLSocketChannel socketChannel = new SSLSocketChannel(sslContext, hostname, port, true);
+ socketChannel.connect();
+
+ commsSession = new SSLSocketChannelCommunicationsSession(socketChannel, destinationUri);
+
+ try {
+ commsSession.setUserDn(socketChannel.getDn());
+ } catch (final CertificateNotYetValidException | CertificateExpiredException ex) {
+ throw new IOException(ex);
+ }
+ } else {
+ final SocketChannel socketChannel = SocketChannel.open(new InetSocketAddress(hostname, port));
+ commsSession = new SocketChannelCommunicationsSession(socketChannel, destinationUri);
+ }
+
+ commsSession.getOutput().getOutputStream().write(CommunicationsSession.MAGIC_BYTES);
+ commsSession.setUri(destinationUri);
+ } catch (final IOException ioe) {
+ if (commsSession != null) {
+ commsSession.close();
+ }
+
+ throw ioe;
+ }
+
+ return commsSession;
+ }
+
+ static List<PeerStatus> formulateDestinationList(final ClusterNodeInformation clusterNodeInfo, final TransferDirection direction) {
+ final Collection<NodeInformation> nodeInfoSet = clusterNodeInfo.getNodeInformation();
+ final int numDestinations = Math.max(128, nodeInfoSet.size());
+ final Map<NodeInformation, Integer> entryCountMap = new HashMap<>();
+
+ long totalFlowFileCount = 0L;
+ for (final NodeInformation nodeInfo : nodeInfoSet) {
+ totalFlowFileCount += nodeInfo.getTotalFlowFiles();
+ }
+
+ int totalEntries = 0;
+ for (final NodeInformation nodeInfo : nodeInfoSet) {
+ final int flowFileCount = nodeInfo.getTotalFlowFiles();
+ // don't allow any node to get more than 80% of the data
+ final double percentageOfFlowFiles = Math.min(0.8D, ((double) flowFileCount / (double) totalFlowFileCount));
+ final double relativeWeighting = (direction == TransferDirection.SEND) ? (1 - percentageOfFlowFiles) : percentageOfFlowFiles;
+ final int entries = Math.max(1, (int) (numDestinations * relativeWeighting));
+
+ entryCountMap.put(nodeInfo, Math.max(1, entries));
+ totalEntries += entries;
+ }
+
+ final List<PeerStatus> destinations = new ArrayList<>(totalEntries);
+ for (int i = 0; i < totalEntries; i++) {
+ destinations.add(null);
+ }
+ for (final Map.Entry<NodeInformation, Integer> entry : entryCountMap.entrySet()) {
+ final NodeInformation nodeInfo = entry.getKey();
+ final int numEntries = entry.getValue();
+
+ int skipIndex = numEntries;
+ for (int i = 0; i < numEntries; i++) {
+ int n = (skipIndex * i);
+ while (true) {
+ final int index = n % destinations.size();
+ PeerStatus status = destinations.get(index);
+ if (status == null) {
+ final PeerDescription description = new PeerDescription(nodeInfo.getHostname(), nodeInfo.getSiteToSitePort(), nodeInfo.isSiteToSiteSecure());
+ status = new PeerStatus(description, nodeInfo.getTotalFlowFiles());
+ destinations.set(index, status);
+ break;
+ } else {
+ n++;
+ }
+ }
+ }
+ }
+
+ final StringBuilder distributionDescription = new StringBuilder();
+ distributionDescription.append("New Weighted Distribution of Nodes:");
+ for (final Map.Entry<NodeInformation, Integer> entry : entryCountMap.entrySet()) {
+ final double percentage = entry.getValue() * 100D / (double) destinations.size();
+ distributionDescription.append("\n").append(entry.getKey()).append(" will receive ").append(percentage).append("% of data");
+ }
+ logger.info(distributionDescription.toString());
+
+ // Jumble the list of destinations.
+ return destinations;
+ }
+
+ private void cleanupExpiredSockets() {
+ for (final BlockingQueue<EndpointConnection> connectionQueue : connectionQueueMap.values()) {
+ final List<EndpointConnection> connections = new ArrayList<>();
+
+ EndpointConnection connection;
+ while ((connection = connectionQueue.poll()) != null) {
+ // If the socket has not been used in 10 seconds, shut it down.
+ final long lastUsed = connection.getLastTimeUsed();
+ if (lastUsed < System.currentTimeMillis() - idleExpirationMillis) {
+ try {
+ connection.getSocketClientProtocol().shutdown(connection.getPeer());
+ } catch (final Exception e) {
+ logger.debug("Failed to shut down {} using {} due to {}",
+ new Object[]{connection.getSocketClientProtocol(), connection.getPeer(), e});
+ }
+
+ terminate(connection);
+ } else {
+ connections.add(connection);
+ }
+ }
+
+ connectionQueue.addAll(connections);
+ }
+ }
+
+ public void shutdown() {
+ shutdown = true;
+ taskExecutor.shutdown();
+ peerTimeoutExpirations.clear();
+
+ for (final EndpointConnection conn : activeConnections) {
+ conn.getPeer().getCommunicationsSession().interrupt();
+ }
+
+ for (final BlockingQueue<EndpointConnection> connectionQueue : connectionQueueMap.values()) {
+ EndpointConnection state;
+ while ((state = connectionQueue.poll()) != null) {
+ cleanup(state.getSocketClientProtocol(), state.getPeer());
+ }
+ }
+ }
+
+ public void terminate(final EndpointConnection connection) {
+ cleanup(connection.getSocketClientProtocol(), connection.getPeer());
+ }
+
+ private void refreshPeers() {
+ final PeerStatusCache existingCache = peerStatusCache;
+ if (existingCache != null && (existingCache.getTimestamp() + PEER_CACHE_MILLIS > System.currentTimeMillis())) {
+ return;
+ }
+
+ try {
+ final Set<PeerStatus> statuses = fetchRemotePeerStatuses();
+ peerStatusCache = new PeerStatusCache(statuses);
+ logger.info("{} Successfully refreshed Peer Status; remote instance consists of {} peers", this, statuses.size());
+ } catch (Exception e) {
+ warn("{} Unable to refresh Remote Group's peers due to {}", this, e);
+ if (logger.isDebugEnabled()) {
+ logger.warn("", e);
+ }
+ }
+ }
+
+ public String getInputPortIdentifier(final String portName) throws IOException {
+ return getPortIdentifier(portName, inputPortMap);
+ }
+
+ public String getOutputPortIdentifier(final String portName) throws IOException {
+ return getPortIdentifier(portName, outputPortMap);
+ }
+
+ private String getPortIdentifier(final String portName, final Map<String, String> portMap) throws IOException {
+ String identifier;
+ remoteInfoReadLock.lock();
+ try {
+ identifier = portMap.get(portName);
+ } finally {
+ remoteInfoReadLock.unlock();
+ }
+
+ if (identifier != null) {
+ return identifier;
+ }
+
+ refreshRemoteInfo();
+
+ remoteInfoReadLock.lock();
+ try {
+ return portMap.get(portName);
+ } finally {
+ remoteInfoReadLock.unlock();
+ }
+ }
+
+ private ControllerDTO refreshRemoteInfo() throws IOException {
+ final boolean webInterfaceSecure = clusterUrl.toString().startsWith("https");
+ final NiFiRestApiUtil utils = new NiFiRestApiUtil(webInterfaceSecure ? sslContext : null);
+ final ControllerDTO controller = utils.getController(apiUri + "/controller", commsTimeout);
+
+ remoteInfoWriteLock.lock();
+ try {
+ this.siteToSitePort = controller.getRemoteSiteListeningPort();
+ this.siteToSiteSecure = controller.isSiteToSiteSecure();
+
+ inputPortMap.clear();
+ for (final PortDTO inputPort : controller.getInputPorts()) {
+ inputPortMap.put(inputPort.getName(), inputPort.getId());
+ }
+
+ outputPortMap.clear();
+ for (final PortDTO outputPort : controller.getOutputPorts()) {
+ outputPortMap.put(outputPort.getName(), outputPort.getId());
+ }
+
+ this.remoteRefreshTime = System.currentTimeMillis();
+ } finally {
+ remoteInfoWriteLock.unlock();
+ }
+
+ return controller;
+ }
+
+ /**
+ * @return the port that the remote instance is listening on for
+ * site-to-site communication, or <code>null</code> if the remote instance
+ * is not configured to allow site-to-site communications.
+ *
+ * @throws IOException if unable to communicate with the remote instance
+ */
+ private Integer getSiteToSitePort() throws IOException {
+ Integer listeningPort;
+ remoteInfoReadLock.lock();
+ try {
+ listeningPort = this.siteToSitePort;
+ if (listeningPort != null && this.remoteRefreshTime > System.currentTimeMillis() - REMOTE_REFRESH_MILLIS) {
+ return listeningPort;
+ }
+ } finally {
+ remoteInfoReadLock.unlock();
+ }
+
+ final ControllerDTO controller = refreshRemoteInfo();
+ listeningPort = controller.getRemoteSiteListeningPort();
+
+ return listeningPort;
+ }
+
+ @Override
+ public String toString() {
+ return "EndpointConnectionPool[Cluster URL=" + clusterUrl + "]";
+ }
+
+ /**
+ * @return {@code true} if the remote instance is configured for secure
+ * site-to-site communications, {@code false} otherwise
+ * @throws IOException if unable to check if secure
+ */
+ public boolean isSecure() throws IOException {
+ remoteInfoReadLock.lock();
+ try {
+ final Boolean secure = this.siteToSiteSecure;
+ if (secure != null && this.remoteRefreshTime > System.currentTimeMillis() - REMOTE_REFRESH_MILLIS) {
+ return secure;
+ }
+ } finally {
+ remoteInfoReadLock.unlock();
+ }
+
+ final ControllerDTO controller = refreshRemoteInfo();
+ final Boolean isSecure = controller.isSiteToSiteSecure();
+ if (isSecure == null) {
+ throw new IOException("Remote NiFi instance " + clusterUrl + " is not currently configured to accept site-to-site connections");
+ }
+
+ return isSecure;
+ }
+
+ private class IdEnrichedRemoteDestination implements RemoteDestination {
+
+ private final RemoteDestination original;
+ private final String identifier;
+
+ public IdEnrichedRemoteDestination(final RemoteDestination original, final String identifier) {
+ this.original = original;
+ this.identifier = identifier;
+ }
+
+ @Override
+ public String getIdentifier() {
+ return identifier;
+ }
+
+ @Override
+ public String getName() {
+ return original.getName();
+ }
+
+ @Override
+ public long getYieldPeriod(final TimeUnit timeUnit) {
+ return original.getYieldPeriod(timeUnit);
+ }
+
+ @Override
+ public boolean isUseCompression() {
+ return original.isUseCompression();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/client/socket/SocketClient.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/client/socket/SocketClient.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/client/socket/SocketClient.java
new file mode 100644
index 0000000..33e4a66
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/client/socket/SocketClient.java
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.client.socket;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.nifi.remote.Communicant;
+import org.apache.nifi.remote.RemoteDestination;
+import org.apache.nifi.remote.Transaction;
+import org.apache.nifi.remote.TransactionCompletion;
+import org.apache.nifi.remote.TransferDirection;
+import org.apache.nifi.remote.client.SiteToSiteClient;
+import org.apache.nifi.remote.client.SiteToSiteClientConfig;
+import org.apache.nifi.remote.protocol.DataPacket;
+import org.apache.nifi.util.ObjectHolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class SocketClient implements SiteToSiteClient {
+
+ private static final Logger logger = LoggerFactory.getLogger(SocketClient.class);
+
+ private final SiteToSiteClientConfig config;
+ private final EndpointConnectionPool pool;
+ private final boolean compress;
+ private final String portName;
+ private final long penalizationNanos;
+ private volatile String portIdentifier;
+ private volatile boolean closed = false;
+
+ public SocketClient(final SiteToSiteClientConfig config) {
+ pool = new EndpointConnectionPool(config.getUrl(),
+ createRemoteDestination(config.getPortIdentifier(), config.getPortName()),
+ (int) config.getTimeout(TimeUnit.MILLISECONDS),
+ (int) config.getIdleConnectionExpiration(TimeUnit.MILLISECONDS),
+ config.getSslContext(), config.getEventReporter(), config.getPeerPersistenceFile());
+
+ this.config = config;
+ this.compress = config.isUseCompression();
+ this.portIdentifier = config.getPortIdentifier();
+ this.portName = config.getPortName();
+ this.penalizationNanos = config.getPenalizationPeriod(TimeUnit.NANOSECONDS);
+ }
+
+ @Override
+ public SiteToSiteClientConfig getConfig() {
+ return config;
+ }
+
+ @Override
+ public boolean isSecure() throws IOException {
+ return pool.isSecure();
+ }
+
+ private String getPortIdentifier(final TransferDirection direction) throws IOException {
+ final String id = this.portIdentifier;
+ if (id != null) {
+ return id;
+ }
+
+ final String portId;
+ if (direction == TransferDirection.SEND) {
+ portId = pool.getInputPortIdentifier(this.portName);
+ } else {
+ portId = pool.getOutputPortIdentifier(this.portName);
+ }
+
+ if (portId == null) {
+ logger.debug("Unable to resolve port [{}] to an identifier", portName);
+ } else {
+ logger.debug("Resolved port [{}] to identifier [{}]", portName, portId);
+ this.portIdentifier = portId;
+ }
+
+ return portId;
+ }
+
+ private RemoteDestination createRemoteDestination(final String portId, final String portName) {
+ return new RemoteDestination() {
+ @Override
+ public String getIdentifier() {
+ return portId;
+ }
+
+ @Override
+ public String getName() {
+ return portName;
+ }
+
+ @Override
+ public long getYieldPeriod(final TimeUnit timeUnit) {
+ return timeUnit.convert(penalizationNanos, TimeUnit.NANOSECONDS);
+ }
+
+ @Override
+ public boolean isUseCompression() {
+ return compress;
+ }
+ };
+ }
+
+ @Override
+ public Transaction createTransaction(final TransferDirection direction) throws IOException {
+ if (closed) {
+ throw new IllegalStateException("Client is closed");
+ }
+ final String portId = getPortIdentifier(direction);
+
+ if (portId == null) {
+ throw new IOException("Could not find Port with name '" + portName + "' for remote NiFi instance");
+ }
+
+ final EndpointConnection connectionState = pool.getEndpointConnection(direction, getConfig());
+ if (connectionState == null) {
+ return null;
+ }
+
+ final Transaction transaction;
+ try {
+ transaction = connectionState.getSocketClientProtocol().startTransaction(
+ connectionState.getPeer(), connectionState.getCodec(), direction);
+ } catch (final Throwable t) {
+ pool.terminate(connectionState);
+ throw new IOException("Unable to create Transaction to communicate with " + connectionState.getPeer(), t);
+ }
+
+ // Wrap the transaction in a new one that will return the EndpointConnectionState back to the pool whenever
+ // the transaction is either completed or canceled.
+ final ObjectHolder<EndpointConnection> connectionStateRef = new ObjectHolder<>(connectionState);
+ return new Transaction() {
+ @Override
+ public void confirm() throws IOException {
+ transaction.confirm();
+ }
+
+ @Override
+ public TransactionCompletion complete() throws IOException {
+ try {
+ return transaction.complete();
+ } finally {
+ final EndpointConnection state = connectionStateRef.get();
+ if (state != null) {
+ pool.offer(connectionState);
+ connectionStateRef.set(null);
+ }
+ }
+ }
+
+ @Override
+ public void cancel(final String explanation) throws IOException {
+ try {
+ transaction.cancel(explanation);
+ } finally {
+ final EndpointConnection state = connectionStateRef.get();
+ if (state != null) {
+ pool.terminate(connectionState);
+ connectionStateRef.set(null);
+ }
+ }
+ }
+
+ @Override
+ public void error() {
+ try {
+ transaction.error();
+ } finally {
+ final EndpointConnection state = connectionStateRef.get();
+ if (state != null) {
+ pool.terminate(connectionState);
+ connectionStateRef.set(null);
+ }
+ }
+ }
+
+ @Override
+ public void send(final DataPacket dataPacket) throws IOException {
+ transaction.send(dataPacket);
+ }
+
+ @Override
+ public void send(final byte[] content, final Map<String, String> attributes) throws IOException {
+ transaction.send(content, attributes);
+ }
+
+ @Override
+ public DataPacket receive() throws IOException {
+ return transaction.receive();
+ }
+
+ @Override
+ public TransactionState getState() throws IOException {
+ return transaction.getState();
+ }
+
+ @Override
+ public Communicant getCommunicant() {
+ return transaction.getCommunicant();
+ }
+ };
+ }
+
+ @Override
+ public void close() throws IOException {
+ closed = true;
+ pool.shutdown();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/cluster/AdaptedNodeInformation.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/cluster/AdaptedNodeInformation.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/cluster/AdaptedNodeInformation.java
new file mode 100644
index 0000000..6ca5812
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/cluster/AdaptedNodeInformation.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.cluster;
+
+public class AdaptedNodeInformation {
+
+ private String hostname;
+ private Integer siteToSitePort;
+ private int apiPort;
+ private boolean isSiteToSiteSecure;
+ private int totalFlowFiles;
+
+ public String getHostname() {
+ return hostname;
+ }
+
+ public void setHostname(String hostname) {
+ this.hostname = hostname;
+ }
+
+ public Integer getSiteToSitePort() {
+ return siteToSitePort;
+ }
+
+ public void setSiteToSitePort(Integer siteToSitePort) {
+ this.siteToSitePort = siteToSitePort;
+ }
+
+ public int getApiPort() {
+ return apiPort;
+ }
+
+ public void setApiPort(int apiPort) {
+ this.apiPort = apiPort;
+ }
+
+ public boolean isSiteToSiteSecure() {
+ return isSiteToSiteSecure;
+ }
+
+ public void setSiteToSiteSecure(boolean isSiteToSiteSecure) {
+ this.isSiteToSiteSecure = isSiteToSiteSecure;
+ }
+
+ public int getTotalFlowFiles() {
+ return totalFlowFiles;
+ }
+
+ public void setTotalFlowFiles(int totalFlowFiles) {
+ this.totalFlowFiles = totalFlowFiles;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/cluster/ClusterNodeInformation.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/cluster/ClusterNodeInformation.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/cluster/ClusterNodeInformation.java
new file mode 100644
index 0000000..1bc83b9
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/cluster/ClusterNodeInformation.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.cluster;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Collection;
+
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.Marshaller;
+import javax.xml.bind.Unmarshaller;
+import javax.xml.bind.annotation.XmlRootElement;
+import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
+
+@XmlRootElement
+public class ClusterNodeInformation {
+
+ private Collection<NodeInformation> nodeInfo;
+
+ private static final JAXBContext JAXB_CONTEXT;
+
+ static {
+ try {
+ JAXB_CONTEXT = JAXBContext.newInstance(ClusterNodeInformation.class);
+ } catch (JAXBException e) {
+ throw new RuntimeException("Unable to create JAXBContext.", e);
+ }
+ }
+
+ public ClusterNodeInformation() {
+ this.nodeInfo = null;
+ }
+
+ public void setNodeInformation(final Collection<NodeInformation> nodeInfo) {
+ this.nodeInfo = nodeInfo;
+ }
+
+ @XmlJavaTypeAdapter(NodeInformationAdapter.class)
+ public Collection<NodeInformation> getNodeInformation() {
+ return nodeInfo;
+ }
+
+ public void marshal(final OutputStream os) throws JAXBException {
+ final Marshaller marshaller = JAXB_CONTEXT.createMarshaller();
+ marshaller.marshal(this, os);
+ }
+
+ public static ClusterNodeInformation unmarshal(final InputStream is) throws JAXBException {
+ final Unmarshaller unmarshaller = JAXB_CONTEXT.createUnmarshaller();
+ return (ClusterNodeInformation) unmarshaller.unmarshal(is);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/cluster/NodeInformation.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/cluster/NodeInformation.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/cluster/NodeInformation.java
new file mode 100644
index 0000000..2041268
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/cluster/NodeInformation.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.cluster;
+
+public class NodeInformation {
+
+ private final String hostname;
+ private final Integer siteToSitePort;
+ private final int apiPort;
+ private final boolean isSiteToSiteSecure;
+ private final int totalFlowFiles;
+
+ public NodeInformation(final String hostname, final Integer siteToSitePort, final int apiPort,
+ final boolean isSiteToSiteSecure, final int totalFlowFiles) {
+ this.hostname = hostname;
+ this.siteToSitePort = siteToSitePort;
+ this.apiPort = apiPort;
+ this.isSiteToSiteSecure = isSiteToSiteSecure;
+ this.totalFlowFiles = totalFlowFiles;
+ }
+
+ public String getHostname() {
+ return hostname;
+ }
+
+ public int getAPIPort() {
+ return apiPort;
+ }
+
+ public Integer getSiteToSitePort() {
+ return siteToSitePort;
+ }
+
+ public boolean isSiteToSiteSecure() {
+ return isSiteToSiteSecure;
+ }
+
+ public int getTotalFlowFiles() {
+ return totalFlowFiles;
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (obj == this) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (!(obj instanceof NodeInformation)) {
+ return false;
+ }
+
+ final NodeInformation other = (NodeInformation) obj;
+ if (!hostname.equals(other.hostname)) {
+ return false;
+ }
+ if (siteToSitePort == null && other.siteToSitePort != null) {
+ return false;
+ }
+ if (siteToSitePort != null && other.siteToSitePort == null) {
+ return false;
+ } else if (siteToSitePort != null && siteToSitePort.intValue() != other.siteToSitePort.intValue()) {
+ return false;
+ }
+ if (apiPort != other.apiPort) {
+ return false;
+ }
+ if (isSiteToSiteSecure != other.isSiteToSiteSecure) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return 83832 + hostname.hashCode() + (siteToSitePort == null ? 8 : siteToSitePort.hashCode()) + apiPort + (isSiteToSiteSecure ? 3829 : 0);
+ }
+
+ @Override
+ public String toString() {
+ return "Node[" + hostname + ":" + apiPort + "]";
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/cluster/NodeInformationAdapter.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/cluster/NodeInformationAdapter.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/cluster/NodeInformationAdapter.java
new file mode 100644
index 0000000..440463c
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/cluster/NodeInformationAdapter.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.cluster;
+
+import javax.xml.bind.annotation.adapters.XmlAdapter;
+
+import org.apache.nifi.remote.cluster.NodeInformation;
+
+public class NodeInformationAdapter extends XmlAdapter<AdaptedNodeInformation, NodeInformation> {
+
+ @Override
+ public NodeInformation unmarshal(final AdaptedNodeInformation adapted) throws Exception {
+ return new NodeInformation(adapted.getHostname(), adapted.getSiteToSitePort(), adapted.getApiPort(), adapted.isSiteToSiteSecure(), adapted.getTotalFlowFiles());
+ }
+
+ @Override
+ public AdaptedNodeInformation marshal(final NodeInformation nodeInformation) throws Exception {
+ final AdaptedNodeInformation adapted = new AdaptedNodeInformation();
+ adapted.setHostname(nodeInformation.getHostname());
+ adapted.setSiteToSitePort(nodeInformation.getSiteToSitePort());
+ adapted.setApiPort(nodeInformation.getAPIPort());
+ adapted.setSiteToSiteSecure(nodeInformation.isSiteToSiteSecure());
+ adapted.setTotalFlowFiles(nodeInformation.getTotalFlowFiles());
+ return adapted;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/codec/FlowFileCodec.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/codec/FlowFileCodec.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/codec/FlowFileCodec.java
new file mode 100644
index 0000000..e79fc47
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/codec/FlowFileCodec.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.codec;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.List;
+
+import org.apache.nifi.remote.VersionedRemoteResource;
+import org.apache.nifi.remote.exception.ProtocolException;
+import org.apache.nifi.remote.exception.TransmissionDisabledException;
+import org.apache.nifi.remote.protocol.DataPacket;
+
+/**
+ * <p>
+ * Provides a mechanism for encoding and decoding FlowFiles as streams so that
+ * they can be transferred remotely.
+ * </p>
+ */
+public interface FlowFileCodec extends VersionedRemoteResource {
+
+ /**
+ * Returns a List of all versions that this codec is able to support, in the
+ * order that they are preferred by the codec
+ *
+ * @return all supported versions
+ */
+ public List<Integer> getSupportedVersions();
+
+ /**
+ * Encodes a DataPacket and its content as a single stream of data and
+ * writes that stream to the output.
+ *
+ * @param dataPacket the data to serialize
+ * @param outStream the stream to write the data to
+ *
+ * @throws IOException if there is a communications issue
+ * @throws TransmissionDisabledException if a user terminates the connection
+ */
+ void encode(DataPacket dataPacket, OutputStream outStream) throws IOException, TransmissionDisabledException;
+
+ /**
+ * Decodes the contents of the InputStream, interpreting the data to
+ * determine the next DataPacket's attributes and content.
+ *
+ * @param stream an InputStream containing DataPacket's content and
+ * attributes
+ *
+ * @return the DataPacket that was created, or <code>null</code> if the
+ * stream was out of data
+ *
+ * @throws IOException if unable to read stream
+ * @throws ProtocolException if the input is malformed
+ * @throws TransmissionDisabledException if a user terminates the connection
+ */
+ DataPacket decode(InputStream stream) throws IOException, ProtocolException, TransmissionDisabledException;
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/codec/StandardFlowFileCodec.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/codec/StandardFlowFileCodec.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/codec/StandardFlowFileCodec.java
new file mode 100644
index 0000000..0bee537
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/codec/StandardFlowFileCodec.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.codec;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.nifi.remote.StandardVersionNegotiator;
+import org.apache.nifi.remote.VersionNegotiator;
+import org.apache.nifi.remote.exception.ProtocolException;
+import org.apache.nifi.remote.protocol.DataPacket;
+import org.apache.nifi.remote.util.StandardDataPacket;
+import org.apache.nifi.stream.io.StreamUtils;
+
+public class StandardFlowFileCodec implements FlowFileCodec {
+
+ public static final int MAX_NUM_ATTRIBUTES = 25000;
+
+ public static final String DEFAULT_FLOWFILE_PATH = "./";
+
+ private final VersionNegotiator versionNegotiator;
+
+ public StandardFlowFileCodec() {
+ versionNegotiator = new StandardVersionNegotiator(1);
+ }
+
+ @Override
+ public void encode(final DataPacket dataPacket, final OutputStream encodedOut) throws IOException {
+ final DataOutputStream out = new DataOutputStream(encodedOut);
+
+ final Map<String, String> attributes = dataPacket.getAttributes();
+ out.writeInt(attributes.size());
+ for (final Map.Entry<String, String> entry : attributes.entrySet()) {
+ writeString(entry.getKey(), out);
+ writeString(entry.getValue(), out);
+ }
+
+ out.writeLong(dataPacket.getSize());
+
+ final InputStream in = dataPacket.getData();
+ StreamUtils.copy(in, encodedOut);
+ encodedOut.flush();
+ }
+
+ @Override
+ public DataPacket decode(final InputStream stream) throws IOException, ProtocolException {
+ final DataInputStream in = new DataInputStream(stream);
+
+ final int numAttributes;
+ try {
+ numAttributes = in.readInt();
+ } catch (final EOFException e) {
+ // we're out of data.
+ return null;
+ }
+
+ // This is here because if the stream is not properly formed, we could get up to Integer.MAX_VALUE attributes, which will
+ // generally result in an OutOfMemoryError.
+ if (numAttributes > MAX_NUM_ATTRIBUTES) {
+ throw new ProtocolException("FlowFile exceeds maximum number of attributes with a total of " + numAttributes);
+ }
+
+ final Map<String, String> attributes = new HashMap<>(numAttributes);
+ for (int i = 0; i < numAttributes; i++) {
+ final String attrName = readString(in);
+ final String attrValue = readString(in);
+ attributes.put(attrName, attrValue);
+ }
+
+ final long numBytes = in.readLong();
+
+ return new StandardDataPacket(attributes, stream, numBytes);
+ }
+
+ private void writeString(final String val, final DataOutputStream out) throws IOException {
+ final byte[] bytes = val.getBytes("UTF-8");
+ out.writeInt(bytes.length);
+ out.write(bytes);
+ }
+
+ private String readString(final DataInputStream in) throws IOException {
+ final int numBytes = in.readInt();
+ final byte[] bytes = new byte[numBytes];
+ StreamUtils.fillBuffer(in, bytes, true);
+ return new String(bytes, "UTF-8");
+ }
+
+ @Override
+ public List<Integer> getSupportedVersions() {
+ return versionNegotiator.getSupportedVersions();
+ }
+
+ @Override
+ public VersionNegotiator getVersionNegotiator() {
+ return versionNegotiator;
+ }
+
+ @Override
+ public String toString() {
+ return "Standard FlowFile Codec, Version " + versionNegotiator.getVersion();
+ }
+
+ @Override
+ public String getResourceName() {
+ return "StandardFlowFileCodec";
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/exception/HandshakeException.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/exception/HandshakeException.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/exception/HandshakeException.java
new file mode 100644
index 0000000..198aaef
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/exception/HandshakeException.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.exception;
+
+import java.io.IOException;
+
+/**
+ * A HandshakeException occurs when the client and the remote NiFi instance do
+ * not agree on some condition during the handshake. For example, if the NiFi
+ * instance does not recognize one of the parameters that the client passes
+ * during the Handshaking phase.
+ */
+public class HandshakeException extends IOException {
+
+ private static final long serialVersionUID = 178192341908726L;
+
+ public HandshakeException(final String message) {
+ super(message);
+ }
+
+ public HandshakeException(final Throwable cause) {
+ super(cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/exception/PortNotRunningException.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/exception/PortNotRunningException.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/exception/PortNotRunningException.java
new file mode 100644
index 0000000..09fc05c
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/exception/PortNotRunningException.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.exception;
+
+/**
+ * PortNotRunningException occurs when the remote NiFi instance reports that the
+ * Port that the client is attempting to communicate with is not currently
+ * running and therefore communications with that Port are not allowed.
+ */
+public class PortNotRunningException extends ProtocolException {
+
+ private static final long serialVersionUID = -2790940982005516375L;
+
+ public PortNotRunningException(final String message) {
+ super(message);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/exception/ProtocolException.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/exception/ProtocolException.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/exception/ProtocolException.java
new file mode 100644
index 0000000..cc6ae50
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/exception/ProtocolException.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.exception;
+
+import java.io.IOException;
+
+/**
+ * A ProtocolException occurs when unexpected data is received, for example an
+ * invalid Response Code.
+ */
+public class ProtocolException extends IOException {
+
+ private static final long serialVersionUID = 5763900324505818495L;
+
+ public ProtocolException(final String message, final Throwable cause) {
+ super(message, cause);
+ }
+
+ public ProtocolException(final String message) {
+ super(message);
+ }
+
+ public ProtocolException(final Throwable cause) {
+ super(cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/exception/UnknownPortException.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/exception/UnknownPortException.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/exception/UnknownPortException.java
new file mode 100644
index 0000000..4249075
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/exception/UnknownPortException.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.exception;
+
+/**
+ * An UnknownPortException indicates that the remote NiFi instance has reported
+ * that the endpoint that the client attempted to communicate with does not
+ * exist.
+ */
+public class UnknownPortException extends ProtocolException {
+
+ private static final long serialVersionUID = -2790940982005516375L;
+
+ public UnknownPortException(final String message) {
+ super(message);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelCommunicationsSession.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelCommunicationsSession.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelCommunicationsSession.java
new file mode 100644
index 0000000..6180c3c
--- /dev/null
+++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/io/socket/SocketChannelCommunicationsSession.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.io.socket;
+
+import java.io.IOException;
+import java.nio.channels.SocketChannel;
+
+import org.apache.nifi.remote.AbstractCommunicationsSession;
+
+public class SocketChannelCommunicationsSession extends AbstractCommunicationsSession {
+
+ private final SocketChannel channel;
+ private final SocketChannelInput request;
+ private final SocketChannelOutput response;
+ private int timeout = 30000;
+
+ public SocketChannelCommunicationsSession(final SocketChannel socketChannel, final String uri) throws IOException {
+ super(uri);
+ request = new SocketChannelInput(socketChannel);
+ response = new SocketChannelOutput(socketChannel);
+ channel = socketChannel;
+ socketChannel.configureBlocking(false);
+ }
+
+ @Override
+ public boolean isClosed() {
+ return !channel.isConnected();
+ }
+
+ @Override
+ public SocketChannelInput getInput() {
+ return request;
+ }
+
+ @Override
+ public SocketChannelOutput getOutput() {
+ return response;
+ }
+
+ @Override
+ public void setTimeout(final int millis) throws IOException {
+ request.setTimeout(millis);
+ response.setTimeout(millis);
+ this.timeout = millis;
+ }
+
+ @Override
+ public int getTimeout() throws IOException {
+ return timeout;
+ }
+
+ @Override
+ public void close() throws IOException {
+ IOException suppressed = null;
+
+ try {
+ request.consume();
+ } catch (final IOException ioe) {
+ suppressed = ioe;
+ }
+
+ try {
+ channel.close();
+ } catch (final IOException ioe) {
+ if (suppressed != null) {
+ ioe.addSuppressed(suppressed);
+ }
+
+ throw ioe;
+ }
+
+ if (suppressed != null) {
+ throw suppressed;
+ }
+ }
+
+ @Override
+ public boolean isDataAvailable() {
+ return request.isDataAvailable();
+ }
+
+ @Override
+ public long getBytesWritten() {
+ return response.getBytesWritten();
+ }
+
+ @Override
+ public long getBytesRead() {
+ return request.getBytesRead();
+ }
+
+ @Override
+ public void interrupt() {
+ request.interrupt();
+ response.interrupt();
+ }
+}
[50/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/LICENSE
----------------------------------------------------------------------
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..59075cd
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,479 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+APACHE NIFI SUBCOMPONENTS:
+
+The Apache NiFi project contains subcomponents with separate copyright
+notices and license terms. Your use of the source code for the these
+subcomponents is subject to the terms and conditions of the following
+licenses.
+
+This product bundles source from 'Asciidoctor'. Specifically the 'asciidoc-mod.css'.
+The source is available under an MIT LICENSE.
+
+ Copyright (C) 2012-2015 Dan Allen, Ryan Waldron and the Asciidoctor Project
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+
+This product bundles 'Javascript D3 Library' which is available under a
+"3-clause BSD" license.
+
+ Copyright (c) 2010-2014, Michael Bostock
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+ * The name Michael Bostock may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ DISCLAIMED. IN NO EVENT SHALL MICHAEL BOSTOCK BE LIABLE FOR ANY DIRECT,
+ INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+This product bundles 'CodeMirror' which is available under an MIT style license.
+
+ Copyright (C) 2014 by Marijn Haverbeke <ma...@gmail.com> and others
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+
+This product bundles 'JQuery' which is available under and MIT style license.
+ (c) 2005, 2014 jQuery Foundation, Inc.
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+
+This product bundles 'JQuery Event Drag' which is available under an MIT style
+license.
+ Copyright (c) 2008-2015 ThreeDubMedia
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+
+This product bundles 'jQuery Form Plugin' which is available under either the MIT
+or GPL license. The license in effect here is the MIT license
+
+ Copyright 2006-2013 (c) M. Alsup
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+This product bundles 'jQuery UI' which is available under an MIT style license.
+For details see http://jqueryui.com
+
+ Copyright 2014 jQuery Foundation and other contributors
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+This product bundles 'SlickGrid v2.2' which is available under an MIT style license.
+
+ Copyright (c) 2010 Michael Leibman, http://github.com/mleibman/slickgrid
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+This product bundles 'qTip2' which is available under an MIT style license.
+For details see http://qtip2.com
+
+ Copyright (c) 2012 Craig Michael Thompson
+
+ Permission is hereby granted, free of charge, to any person
+ obtaining a copy of this software and associated documentation
+ files (the "Software"), to deal in the Software without
+ restriction, including without limitation the rights to use,
+ copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the
+ Software is furnished to do so, subject to the following
+ conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ OTHER DEALINGS IN THE SOFTWARE.
+
+This product bundles 'jQuery MiniColors' which is available under the MIT License.
+For details see http://www.abeautifulsite.net/
+
+ Copyright Cory LaViska for A Beautiful Site, LLC. (http://www.abeautifulsite.net/)
+
+ Permission is hereby granted, free of charge, to any person
+ obtaining a copy of this software and associated documentation
+ files (the "Software"), to deal in the Software without
+ restriction, including without limitation the rights to use,
+ copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the
+ Software is furnished to do so, subject to the following
+ conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ OTHER DEALINGS IN THE SOFTWARE.
+
+This product bundles 'json2.js' which is available in the 'public domain'.
+ For details see https://github.com/douglascrockford/JSON-js
+
+This product bundles 'reset.css' which is available in the 'public domain'.
+ For details see http://meyerweb.com/eric/tools/css/reset/
+
+This product bundles HexViewJS available under an MIT License
+
+ Copyright (c) 2010 Nick McVeity <nm...@gmail.com>
+
+ Permission is hereby granted, free of charge, to any person
+ obtaining a copy of this software and associated documentation
+ files (the "Software"), to deal in the Software without restriction,
+ including without limitation the rights to use, copy, modify, merge,
+ publish, distribute, sublicense, and/or sell copies of the Software,
+ and to permit persons to whom the Software is furnished to do so,
+ subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/NOTICE
----------------------------------------------------------------------
diff --git a/NOTICE b/NOTICE
new file mode 100644
index 0000000..9686fba
--- /dev/null
+++ b/NOTICE
@@ -0,0 +1,6 @@
+Apache NiFi
+Copyright 2014-2015 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index c71bb0f..78f2242 100644
--- a/README.md
+++ b/README.md
@@ -18,49 +18,105 @@ Apache NiFi is an easy to use, powerful, and reliable system to process and dist
## Table of Contents
+- [Features](#features)
+- [Requirements](#requirements)
- [Getting Started](#getting-started)
+- [Getting Help](#getting-help)
+- [Documentation](#documentation)
- [License](#license)
+- [Export Control] (#export-control)
+
+## Features
+
+Apache NiFi was made for dataflow. It supports highly configurable directed graphs of data routing, transformation, and system mediation logic. Some of its key features include:
+
+- Web-based user interface
+ - Seamless experience for design, control, and monitoring
+- Highly configurable
+ - Loss tolerant vs guaranteed delivery
+ - Low latency vs high throughput
+ - Dynamic prioritization
+ - Flows can be modified at runtime
+ - Back pressure
+- Data Provenance
+ - Track dataflow from beginning to end
+- Designed for extension
+ - Build your own processors and more
+ - Enables rapid development and effective testing
+- Secure
+ - SSL, SSH, HTTPS, encrypted content, etc...
+ - Pluggable role-based authentication/authorization
+
+## Requirements
+* JDK 1.7 or higher
+* Apache Maven 3.0.5 or higher
## Getting Started
-- Read through the [quickstart guide for development](http://nifi.apache.org/quickstart.html).
+- Read through the [quickstart guide for development](http://nifi.incubator.apache.org/quickstart.html).
It will include information on getting a local copy of the source, give pointers on issue
tracking, and provide some warnings about common problems with development environments.
- For a more comprehensive guide to development and information about contributing to the project
- read through the [NiFi Developer's Guide](http://nifi.apache.org/developer-guide.html).
-- Optional: Build supporting modules. This should only be needed if the current 'nifi' module is in
- the process of updating to a new version of either the 'nifi-parent' or 'nifi-nar-maven-plugin'
- artifacts.
-
- If in doubt, just skip to building the main nifi project. If the build fails, come back here and
- figure out which optional step you are missing; each entry below will give an example of the
- errors you'll receive if that step needs to be followed. The version numbers may change but the
- error text should still look very familiar.
- - Install the nifi-parent pom. Change directory to 'nifi-parent' and follow the directions found
- there in [README.md](nifi-parent/README.md).
-
- If you don't build the nifi-parent pom and the main nifi code relies on an unreleased version
- you'll see an erorr like the following:
-
- [ERROR] Non-resolvable parent POM: Could not find artifact
- org.apache.nifi:nifi-parent:pom:1.0.0-SNAPSHOT in example.snapshots.repo
- (https://repository.example.com/content/repositories/snapshots) and
- 'parent.relativePath' points at no local POM @ line 18, column 13 -> [Help 2]
- - Build the nifi-nar-maven-plugin. Change directory to 'nifi-nar-maven-plugin' and
- follow the directions found there in [README.md](nifi-nar-maven-plugin/README.md).
-
- If you don't build the nifi-nar-maven-plugin and the main nifi code relies on an unreleased
- version you'll see an error like the following:
-
- [ERROR] Unresolveable build extension: Plugin
- org.apache.nifi:nifi-nar-maven-plugin:1.0.1-SNAPSHOT or one of its
- dependencies could not be resolved: Could not find artifact
- org.apache.nifi:nifi-nar-maven-plugin:jar:1.0.1-SNAPSHOT -> [Help 2]
-- Build nifi. Change directory to 'nifi' and follow the directions found there in
- [README.md](nifi/README.md).
-- Run NiFi. The directions found in the [README.md](nifi/README.md) file within the 'nifi' module
- will also include how to run an instance of NiFi. For help on how to build your first data flow,
- see the [NiFi User Guide](http://nifi.apache.org/docs/nifi-docs/user-guide.html).
+ read through the [NiFi Developer's Guide](http://nifi.incubator.apache.org/developer-guide.html).
+
+To build:
+- Execute `mvn clean install` or for parallel build execute `mvn -T 2.0C clean install`. On a
+ modest development laptop that is a couple of years old, the latter build takes a bit under ten
+ minutes. After a large amount of output you should eventually see a success message.
+
+ laptop:nifi fhampton$ mvn -T 2.0C clean install
+ [INFO] Scanning for projects...
+ [INFO] Inspecting build with total of 115 modules...
+ ...tens of thousands of lines elided...
+ [INFO] ------------------------------------------------------------------------
+ [INFO] BUILD SUCCESS
+ [INFO] ------------------------------------------------------------------------
+ [INFO] Total time: 09:24 min (Wall Clock)
+ [INFO] Finished at: 2015-04-30T00:30:36-05:00
+ [INFO] Final Memory: 173M/1359M
+ [INFO] ------------------------------------------------------------------------
+
+To deploy:
+- Change directory to 'nifi-assembly'. In the target directory, there should be a build of nifi.
+
+ laptop:nifi fhampton$ cd nifi-assembly
+ laptop:nifi-assembly fhampton$ ls -lhd target/nifi*
+ drwxr-xr-x 3 fhampton staff 102B Apr 30 00:29 target/nifi-0.1.0-SNAPSHOT-bin
+ -rw-r--r-- 1 fhampton staff 144M Apr 30 00:30 target/nifi-0.1.0-SNAPSHOT-bin.tar.gz
+ -rw-r--r-- 1 fhampton staff 144M Apr 30 00:30 target/nifi-0.1.0-SNAPSHOT-bin.zip
+
+- For testing ongoing development you could use the already unpacked build present in the directory
+ named "nifi-*version*-bin", where *version* is the current project version. To deploy in another
+ location make use of either the tarball or zipfile and unpack them wherever you like. The
+ distribution will be within a common parent directory named for the version.
+
+ laptop:nifi-assembly fhampton$ mkdir ~/example-nifi-deploy
+ laptop:nifi-assembly fhampton$ tar xzf target/nifi-*-bin.tar.gz -C ~/example-nifi-deploy
+ laptop:nifi-assembly fhampton$ ls -lh ~/example-nifi-deploy/
+ total 0
+ drwxr-xr-x 10 fhampton staff 340B Apr 30 01:06 nifi-0.1.0-SNAPSHOT
+
+To run NiFi:
+- Change directory to the location where you installed NiFi and run it.
+
+ laptop:~ fhampton$ cd ~/example-nifi-deploy/nifi-*
+ laptop:nifi-0.1.0-SNAPSHOT fhampton$ ./bin/nifi.sh start
+
+- Direct your browser to http://localhost:8080/nifi/ and you should see a screen like this screenshot:
+ 
+
+- For help building your first data flow see the [NiFi User Guide](http://nifi.apache.org/docs/nifi-docs/user-guide.html)
+
+- If you are testing ongoing development, you will likely want to stop your instance.
+
+ laptop:~ fhampton$ cd ~/example-nifi-deploy/nifi-*
+ laptop:nifi-0.1.0-SNAPSHOT fhampton$ ./bin/nifi.sh stop
+
+## Getting Help
+If you have questions, you can reach out to our mailing list: dev@nifi.apache.org
+([archive](http://mail-archives.apache.org/mod_mbox/nifi-dev)).
+We're also often available in IRC: #nifi on
+[irc.freenode.net](http://webchat.freenode.net/?channels=#nifi).
## Documentation
@@ -83,3 +139,32 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
+## Export Control
+
+This distribution includes cryptographic software. The country in which you
+currently reside may have restrictions on the import, possession, use, and/or
+re-export to another country, of encryption software. BEFORE using any
+encryption software, please check your country's laws, regulations and
+policies concerning the import, possession, or use, and re-export of encryption
+software, to see if this is permitted. See <http://www.wassenaar.org/> for more
+information.
+
+The U.S. Government Department of Commerce, Bureau of Industry and Security
+(BIS), has classified this software as Export Commodity Control Number (ECCN)
+5D002.C.1, which includes information security software using or performing
+cryptographic functions with asymmetric algorithms. The form and manner of this
+Apache Software Foundation distribution makes it eligible for export under the
+License Exception ENC Technology Software Unrestricted (TSU) exception (see the
+BIS Export Administration Regulations, Section 740.13) for both object code and
+source code.
+
+The following provides more details on the included cryptographic software:
+
+Apache NiFi uses BouncyCastle, Jasypt, JCraft Inc., and the built-in
+java cryptography libraries for SSL, SSH, and the protection
+of sensitive configuration parameters. See
+http://bouncycastle.org/about.html
+http://www.jasypt.org/faq.html
+http://jcraft.com/c-info.html
+http://www.oracle.com/us/products/export/export-regulations-345813.html
+for more details on each of these libraries cryptography features.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/.gitignore
----------------------------------------------------------------------
diff --git a/nifi-api/.gitignore b/nifi-api/.gitignore
new file mode 100755
index 0000000..19f2e00
--- /dev/null
+++ b/nifi-api/.gitignore
@@ -0,0 +1,2 @@
+/target
+/target
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-api/pom.xml b/nifi-api/pom.xml
new file mode 100644
index 0000000..ffdf192
--- /dev/null
+++ b/nifi-api/pom.xml
@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>nifi-api</artifactId>
+ <packaging>jar</packaging>
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/DynamicProperties.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/DynamicProperties.java b/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/DynamicProperties.java
new file mode 100644
index 0000000..902f5e9
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/DynamicProperties.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.annotation.behavior;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Indicates that a component has more than one dynamic property
+ *
+ */
+@Documented
+@Target({ElementType.TYPE})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+public @interface DynamicProperties {
+
+ public DynamicProperty[] value();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/DynamicProperty.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/DynamicProperty.java b/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/DynamicProperty.java
new file mode 100644
index 0000000..dbb34b6
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/DynamicProperty.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.annotation.behavior;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import org.apache.nifi.components.ConfigurableComponent;
+
+/**
+ * An annotation that may be placed on a {@link ConfigurableComponent} to
+ * indicate that it supports a dynamic property.
+ *
+ */
+@Documented
+@Target({ElementType.TYPE})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+public @interface DynamicProperty {
+
+ public String name();
+
+ public boolean supportsExpressionLanguage() default false;
+
+ public String value();
+
+ public String description();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/DynamicRelationship.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/DynamicRelationship.java b/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/DynamicRelationship.java
new file mode 100644
index 0000000..4129201
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/DynamicRelationship.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.annotation.behavior;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.processor.Processor;
+import org.apache.nifi.processor.Relationship;
+
+/**
+ * Annotation to indicate that a {@link Processor} supports dynamic
+ * relationship. A dynamic {@link Relationship} is one where the relationship is
+ * generated based on a user defined {@link PropertyDescriptor}
+ *
+ */
+@Documented
+@Target({ElementType.TYPE})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+public @interface DynamicRelationship {
+
+ public String name();
+
+ public String description();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/EventDriven.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/EventDriven.java b/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/EventDriven.java
new file mode 100644
index 0000000..c548b59
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/EventDriven.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.annotation.behavior;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * <p>
+ * Annotation that may be placed on a Processor that indicates to the framework
+ * that the Processor is eligible to be scheduled to run based on the occurrence
+ * of an "Event" (e.g., when a FlowFile is enqueued in an incoming Connection),
+ * rather than being triggered periodically.
+ * </p>
+ *
+ * <p>
+ * This Annotation should not be used in conjunction with
+ * {@link TriggerSerially} or {@link TriggerWhenEmpty}. If this Annotation is
+ * used with either of these other Annotations, the Processor will not be
+ * eligible to be scheduled in Event-Driven mode.
+ * </p>
+ *
+ */
+@Documented
+@Target({ElementType.TYPE})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+public @interface EventDriven {
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/ReadsAttribute.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/ReadsAttribute.java b/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/ReadsAttribute.java
new file mode 100644
index 0000000..fb9e2ab
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/ReadsAttribute.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.annotation.behavior;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Annotation that may be placed on a
+ * {@link org.apache.nifi.processor.Processor Processor} indicating that this
+ * processor reads a specific FlowFile attribute.
+ *
+ */
+@Documented
+@Target({ElementType.TYPE})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+public @interface ReadsAttribute {
+
+ public String attribute();
+
+ public String description() default "";
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/ReadsAttributes.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/ReadsAttributes.java b/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/ReadsAttributes.java
new file mode 100644
index 0000000..2610e1c
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/ReadsAttributes.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.annotation.behavior;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Annotation that may be placed on a
+ * {@link org.apache.nifi.processor.Processor Processor} indicating that this
+ * processor reads specific FlowFile attributes.
+ *
+ */
+@Documented
+@Target({ElementType.TYPE})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+public @interface ReadsAttributes {
+
+ public ReadsAttribute[] value();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/SideEffectFree.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/SideEffectFree.java b/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/SideEffectFree.java
new file mode 100644
index 0000000..9a64f4b
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/SideEffectFree.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.annotation.behavior;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Marker annotation a {@link org.apache.nifi.processor.Processor Processor}
+ * implementation can use to indicate that its operations on FlowFiles can be
+ * safely repeated across process sessions. If a processor has this annotation
+ * and it allows the framework to manage session commit and rollback then the
+ * framework may elect to cascade a
+ * {@link org.apache.nifi.processor.ProcessSession ProcessSession} given to this
+ * processor's onTrigger method to the onTrigger method of another processor. It
+ * can do this knowing that if something fails along a series of processors
+ * using this same session that it can all be safely rolled back without any ill
+ * effects on external services which could not be rolled back and thus all the
+ * processes could be safely repeated (implied idempotent behavior).
+ *
+ */
+@Documented
+@Target({ElementType.TYPE})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+public @interface SideEffectFree {
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/SupportsBatching.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/SupportsBatching.java b/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/SupportsBatching.java
new file mode 100644
index 0000000..807e637
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/SupportsBatching.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.annotation.behavior;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Marker annotation a Processor implementation can use to indicate that users
+ * should be able to supply a Batch Duration for the Processor. If a Processor
+ * uses this annotation, it is allowing the Framework to batch
+ * {@link nifi.processor.ProcessSession ProcessSession}s' commits, as well as
+ * allowing the Framework to return the same ProcessSession multiple times from
+ * subsequent calls to
+ * {@link nifi.processor.ProcessSessionFactory ProcessSessionFactory}.
+ * {@link nifi.processor.ProcessSessionFactory#createSession() createSession()}.
+ *
+ * When this Annotation is used, it is important to note that calls to
+ * {@link nifi.processor.ProcessSession#commit() ProcessSession.commit()} may
+ * not provide a guarantee that the data has been safely stored in NiFi's
+ * Content Repository or FlowFile Repository. Therefore, it is not appropriate,
+ * for instance, to use this annotation if the Processor will call
+ * ProcessSession.commit() to ensure data is persisted before deleting the data
+ * from a remote source.
+ *
+ */
+@Documented
+@Target({ElementType.TYPE})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+public @interface SupportsBatching {
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/TriggerSerially.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/TriggerSerially.java b/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/TriggerSerially.java
new file mode 100644
index 0000000..974e358
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/TriggerSerially.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.annotation.behavior;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Marker annotation a {@link org.apache.nifi.processor.Processor Processor}
+ * implementation can use to indicate that the Processor is not safe for
+ * concurrent execution of its onTrigger() method. By default, Processors are
+ * assumed to be safe for concurrent execution.
+ *
+ */
+@Documented
+@Target({ElementType.TYPE})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+public @interface TriggerSerially {
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/TriggerWhenAnyDestinationAvailable.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/TriggerWhenAnyDestinationAvailable.java b/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/TriggerWhenAnyDestinationAvailable.java
new file mode 100644
index 0000000..db115cb
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/TriggerWhenAnyDestinationAvailable.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.annotation.behavior;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Marker annotation a {@link org.apache.nifi.processor.Processor Processor}
+ * implementation can use to indicate that the Processor is to be triggered if
+ * any of its destinations has available space for incoming FlowFiles. By
+ * default, Processors are triggered only when all destinations report that they
+ * have available space (i.e., none of the outgoing Connections is full).
+ *
+ */
+@Documented
+@Target({ElementType.TYPE})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+public @interface TriggerWhenAnyDestinationAvailable {
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/TriggerWhenEmpty.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/TriggerWhenEmpty.java b/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/TriggerWhenEmpty.java
new file mode 100644
index 0000000..f442b80
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/TriggerWhenEmpty.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.annotation.behavior;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Marker annotation a {@link org.apache.nifi.processor.Processor Processor}
+ * implementation can use to indicate that the Processor should still be
+ * triggered even when it has no data in its work queue. By default, Processors
+ * which have no non-self incoming edges will be triggered even if there is no
+ * work in its queue. However, Processors that have non-self incoming edges will
+ * only be triggered if they have work in their queue or they present this
+ * annotation.
+ *
+ */
+@Documented
+@Target({ElementType.TYPE})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+public @interface TriggerWhenEmpty {
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/WritesAttribute.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/WritesAttribute.java b/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/WritesAttribute.java
new file mode 100644
index 0000000..841cf40
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/WritesAttribute.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.annotation.behavior;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Annotation that may be placed on a
+ * {@link org.apache.nifi.processor.Processor Processor} indicating that this
+ * processor writes/updates a specific FlowFile attribute.
+ *
+ */
+@Documented
+@Target({ElementType.TYPE})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+public @interface WritesAttribute {
+
+ public String attribute();
+
+ public String description() default "";
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/WritesAttributes.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/WritesAttributes.java b/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/WritesAttributes.java
new file mode 100644
index 0000000..3537e72
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/WritesAttributes.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.annotation.behavior;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Annotation that may be placed on a
+ * {@link org.apache.nifi.processor.Processor Processor} indicating that this
+ * processor writes/updates specific FlowFile attributes.
+ *
+ */
+@Documented
+@Target({ElementType.TYPE})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+public @interface WritesAttributes {
+
+ public WritesAttribute[] value();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/annotation/documentation/CapabilityDescription.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/annotation/documentation/CapabilityDescription.java b/nifi-api/src/main/java/org/apache/nifi/annotation/documentation/CapabilityDescription.java
new file mode 100644
index 0000000..cff6743
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/annotation/documentation/CapabilityDescription.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.annotation.documentation;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Annotation that may be placed on a {@link org.apache.nifi.processor.Processor Processor},
+ * {@link org.apache.nifi.controller.ControllerService ControllerService}, or
+ * {@link org.apache.nifi.reporting.ReportingTask ReportingTask} allowing for a
+ * description to be provided. This description can be provided to a user in
+ * logs, UI, etc.
+ *
+ */
+@Documented
+@Target({ElementType.TYPE})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+public @interface CapabilityDescription {
+
+ String value();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/annotation/documentation/SeeAlso.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/annotation/documentation/SeeAlso.java b/nifi-api/src/main/java/org/apache/nifi/annotation/documentation/SeeAlso.java
new file mode 100644
index 0000000..35ca3db
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/annotation/documentation/SeeAlso.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.annotation.documentation;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import org.apache.nifi.components.ConfigurableComponent;
+
+/**
+ * Annotation that may be placed on a null {@link org.apache.nifi.processor.Processor Processor},
+ * {@link org.apache.nifi.controller.ControllerService ControllerService}, or
+ * {@link org.apache.nifi.reporting.ReportingTask ReportingTask} that indicates
+ * this component is related to the components listed.
+ *
+ */
+@Documented
+@Target({ElementType.TYPE})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+public @interface SeeAlso {
+
+ public Class<? extends ConfigurableComponent>[] value() default {};
+
+ public String[] classNames() default {};
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/annotation/documentation/Tags.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/annotation/documentation/Tags.java b/nifi-api/src/main/java/org/apache/nifi/annotation/documentation/Tags.java
new file mode 100644
index 0000000..366dd4c
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/annotation/documentation/Tags.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.annotation.documentation;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Annotation that can be applied to a {@link org.apache.nifi.processor.Processor Processor},
+ * {@link org.apache.nifi.controller.ControllerService ControllerService}, or
+ * {@link org.apache.nifi.reporting.ReportingTask ReportingTask} in order to
+ * associate tags (keywords) with the component. These tags do not affect the
+ * component in any way but serve as additional documentation and can be used to
+ * sort/filter Processors.
+ *
+ */
+@Documented
+@Target({ElementType.TYPE})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+public @interface Tags {
+
+ public String[] value();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnAdded.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnAdded.java b/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnAdded.java
new file mode 100644
index 0000000..95edc0a
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnAdded.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.annotation.lifecycle;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * <p>
+ * Marker annotation a {@link org.apache.nifi.processor.Processor Processor},
+ * {@link org.apache.nifi.controller.ControllerService ControllerService}, or
+ * {@link org.apache.nifi.reporting.ReportingTask ReportingTask} implementation
+ * can use to indicate a method should be called whenever the component is added
+ * to the flow. This method will be called once for the entire life of a
+ * component instance.
+ * </p>
+ *
+ * <p>
+ * Methods with this annotation are called without any arguments, as all
+ * settings and properties can be assumed to be the defaults.
+ * </p>
+ *
+ * <p>
+ * If any method annotated with this annotation throws a Throwable, the
+ * component will not be added to the flow.
+ * </p>
+ *
+ */
+@Documented
+@Target({ElementType.METHOD})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+public @interface OnAdded {
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnDisabled.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnDisabled.java b/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnDisabled.java
new file mode 100644
index 0000000..f8ca038
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnDisabled.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.annotation.lifecycle;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import org.apache.nifi.controller.ConfigurationContext;
+
+/**
+ * <p>
+ * Marker annotation a
+ * {@link org.apache.nifi.controller.ControllerService ControllerService} can
+ * use to indicate a method should be called whenever the service is disabled.
+ * </p>
+ *
+ * <p>
+ * Methods using this annotation are permitted to take zero arguments or to take
+ * a single argument of type {@link ConfigurationContext}. If a method with this
+ * annotation throws a Throwable, a log message and bulletin will be issued for
+ * the service, but the service will still be marked as Disabled. The failing
+ * method will not be called again until the service is enabled and disabled again.
+ * This is done in order to prevent a ControllerService from continually failing
+ * in such a way that the service could not be disabled and updated without
+ * restarting the instance of NiFi.
+ * </p>
+ *
+ * <p>
+ * Note that this annotation will be ignored if applied to a ReportingTask or
+ * Processor. For a Controller Service, enabling and disabling are considered
+ * lifecycle events, as the action makes them usable or unusable by other
+ * components. However, for a Processor and a Reporting Task, these are not
+ * lifecycle events but rather a mechanism to allow a component to be excluded
+ * when starting or stopping a group of components.
+ * </p>
+ *
+ */
+@Documented
+@Target({ElementType.METHOD})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+public @interface OnDisabled {
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnEnabled.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnEnabled.java b/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnEnabled.java
new file mode 100644
index 0000000..289b6c4
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnEnabled.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.annotation.lifecycle;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * <p>
+ * Marker annotation a
+ * {@link org.apache.nifi.controller.ControllerService ControllerService} can
+ * use to indicate a method should be called whenever the service is enabled.
+ * Any method that has this annotation will be called every time a user enables
+ * the service. Additionally, each time that NiFi is restarted, if NiFi is
+ * configured to "auto-resume state" and the service is enabled, the method will
+ * be invoked.
+ * </p>
+ *
+ * <p>
+ * Methods using this annotation must take either 0 arguments or a single
+ * argument of type
+ * {@link org.apache.nifi.controller.ConfigurationContext ConfigurationContext}.
+ * </p>
+ *
+ * <p>
+ * If a method with this annotation throws a Throwable, a log message and
+ * bulletin will be issued for the component. In this event, the service will
+ * remain in an 'ENABLING' state and will not be usable. All methods with this
+ * annotation will then be called again after a delay. The service will not be
+ * made available for use until all methods with this annotation have returned
+ * without throwing anything.
+ * </p>
+ *
+ * <p>
+ * Note that this annotation will be ignored if applied to a ReportingTask or
+ * Processor. For a Controller Service, enabling and disabling are considered
+ * lifecycle events, as the action makes them usable or unusable by other
+ * components. However, for a Processor and a Reporting Task, these are not
+ * lifecycle events but rather a mechanism to allow a component to be excluded
+ * when starting or stopping a group of components.
+ * </p>
+ *
+ *
+ */
+@Documented
+@Target({ElementType.METHOD})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+public @interface OnEnabled {
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnRemoved.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnRemoved.java b/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnRemoved.java
new file mode 100644
index 0000000..54817e4
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnRemoved.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.annotation.lifecycle;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import org.apache.nifi.controller.ConfigurationContext;
+import org.apache.nifi.processor.ProcessContext;
+
+/**
+ * <p>
+ * Marker annotation a {@link org.apache.nifi.processor.Processor Processor},
+ * {@link org.apache.nifi.controller.ControllerService ControllerService}, or
+ * {@link org.apache.nifi.reporting.ReportingTask ReportingTask} implementation
+ * can use to indicate a method should be called whenever the component is
+ * removed from the flow. This method will be called once for the entire life of
+ * a component instance. If the method throw any Throwable, that Throwable will
+ * be caught and logged but will not prevent subsequent methods with this
+ * annotation or removal of the component from the flow.
+ * </p>
+ *
+ * <p>
+ * Methods with this annotation are permitted to take no arguments or to take a
+ * single argument. If using a single argument, that argument must be of type
+ * {@link ConfigurationContext} if the component is a ReportingTask or a
+ * ControllerService. If the component is a Processor, then the argument must be
+ * of type {@link ProcessContext}.
+ * </p>
+ *
+ */
+@Documented
+@Target({ElementType.METHOD})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+public @interface OnRemoved {
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnScheduled.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnScheduled.java b/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnScheduled.java
new file mode 100644
index 0000000..8fa752b
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnScheduled.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.annotation.lifecycle;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * <p>
+ * Marker annotation a {@link org.apache.nifi.processor.Processor Processor} or
+ * {@link org.apache.nifi.reporting.ReportingTask ReportingTask} implementation
+ * can use to indicate a method should be called whenever the component is
+ * scheduled to run. This will be called before any call to 'onTrigger' and will
+ * be called once each time a Processor or Reporting Task is scheduled to run.
+ * This occurs in one of two ways: either a user clicks to schedule the
+ * component to run, or NiFi is restarted with the "auto-resume state"
+ * configuration set to true (the default value) and the component is already
+ * running.
+ * </p>
+ *
+ * <p>
+ * Methods using this annotation must take either 0 arguments or a single
+ * argument.
+ * </p>
+ *
+ * <p>
+ * If using 1 argument and the component using the annotation is a Processor,
+ * that argument must be of type
+ * {@link org.apache.nifi.processor.ProcessContext ProcessContext}.
+ * </p>
+ *
+ * <p>
+ * If using 1 argument and the component using the annotation is a Reporting
+ * Task, that argument must be of type
+ * {@link org.apache.nifi.controller.ConfigurationContext ConfigurationContext}.
+ * </p>
+ *
+ * If any method annotated with this annotation throws any Throwable, the
+ * framework will wait a while and then attempt to invoke the method again. This
+ * will continue until the method succeeds, and the component will then be
+ * scheduled to run after this method return successfully.
+ *
+ */
+@Documented
+@Target({ElementType.METHOD})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+public @interface OnScheduled {
+}
[21/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/search/ahocorasick/SearchState.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/search/ahocorasick/SearchState.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/search/ahocorasick/SearchState.java
new file mode 100644
index 0000000..6d36ad0
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/search/ahocorasick/SearchState.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.search.ahocorasick;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.nifi.util.search.SearchTerm;
+
+public class SearchState<T> {
+
+ private Node currentNode;
+ private final Map<SearchTerm<T>, List<Long>> resultMap;
+ private long bytesRead;
+
+ SearchState(final Node rootNode) {
+ resultMap = new HashMap<>(5);
+ currentNode = rootNode;
+ bytesRead = 0L;
+ }
+
+ void incrementBytesRead(final long increment) {
+ bytesRead += increment;
+ }
+
+ void setCurrentNode(final Node curr) {
+ currentNode = curr;
+ }
+
+ public Node getCurrentNode() {
+ return currentNode;
+ }
+
+ public Map<SearchTerm<T>, List<Long>> getResults() {
+ return new HashMap<>(resultMap);
+ }
+
+ void addResult(final SearchTerm matchingTerm) {
+ final List<Long> indexes = (resultMap.containsKey(matchingTerm)) ? resultMap.get(matchingTerm) : new ArrayList<Long>(5);
+ indexes.add(bytesRead);
+ resultMap.put(matchingTerm, indexes);
+ }
+
+ public boolean foundMatch() {
+ return !resultMap.isEmpty();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/timebuffer/EntityAccess.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/timebuffer/EntityAccess.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/timebuffer/EntityAccess.java
new file mode 100644
index 0000000..2b95897
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/timebuffer/EntityAccess.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.timebuffer;
+
+public interface EntityAccess<T> {
+
+ T aggregate(T oldValue, T toAdd);
+
+ T createNew();
+
+ long getTimestamp(T entity);
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/timebuffer/LongEntityAccess.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/timebuffer/LongEntityAccess.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/timebuffer/LongEntityAccess.java
new file mode 100644
index 0000000..193abc6
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/timebuffer/LongEntityAccess.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.timebuffer;
+
+public class LongEntityAccess implements EntityAccess<TimestampedLong> {
+
+ @Override
+ public TimestampedLong aggregate(TimestampedLong oldValue, TimestampedLong toAdd) {
+ if (oldValue == null && toAdd == null) {
+ return new TimestampedLong(0L);
+ } else if (oldValue == null) {
+ return toAdd;
+ } else if (toAdd == null) {
+ return oldValue;
+ }
+
+ return new TimestampedLong(oldValue.getValue() + toAdd.getValue());
+ }
+
+ @Override
+ public TimestampedLong createNew() {
+ return new TimestampedLong(0L);
+ }
+
+ @Override
+ public long getTimestamp(TimestampedLong entity) {
+ return entity == null ? 0L : entity.getTimestamp();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/timebuffer/TimedBuffer.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/timebuffer/TimedBuffer.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/timebuffer/TimedBuffer.java
new file mode 100644
index 0000000..dd8e523
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/timebuffer/TimedBuffer.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.timebuffer;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+public class TimedBuffer<T> {
+
+ private final int numBins;
+ private final EntitySum<T>[] bins;
+ private final EntityAccess<T> entityAccess;
+ private final TimeUnit binPrecision;
+
+ @SuppressWarnings("unchecked")
+ public TimedBuffer(final TimeUnit binPrecision, final int numBins, final EntityAccess<T> accessor) {
+ this.binPrecision = binPrecision;
+ this.numBins = numBins + 1;
+ this.bins = new EntitySum[this.numBins];
+ for (int i = 0; i < this.numBins; i++) {
+ this.bins[i] = new EntitySum<>(binPrecision, numBins, accessor);
+ }
+ this.entityAccess = accessor;
+ }
+
+ public T add(final T entity) {
+ final int binIdx = (int) (binPrecision.convert(System.currentTimeMillis(), TimeUnit.MILLISECONDS) % numBins);
+ final EntitySum<T> sum = bins[binIdx];
+
+ return sum.addOrReset(entity);
+ }
+
+ public T getAggregateValue(final long sinceEpochMillis) {
+ final int startBinIdx = (int) (binPrecision.convert(sinceEpochMillis, TimeUnit.MILLISECONDS) % numBins);
+
+ T total = null;
+ for (int i = 0; i < numBins; i++) {
+ int binIdx = (startBinIdx + i) % numBins;
+ final EntitySum<T> bin = bins[binIdx];
+
+ if (!bin.isExpired()) {
+ total = entityAccess.aggregate(total, bin.getValue());
+ }
+ }
+
+ return total;
+ }
+
+ private static class EntitySum<S> {
+
+ private final EntityAccess<S> entityAccess;
+ private final AtomicReference<S> ref = new AtomicReference<>();
+ private final TimeUnit binPrecision;
+ private final int numConfiguredBins;
+
+ public EntitySum(final TimeUnit binPrecision, final int numConfiguredBins, final EntityAccess<S> aggregator) {
+ this.binPrecision = binPrecision;
+ this.entityAccess = aggregator;
+ this.numConfiguredBins = numConfiguredBins;
+ }
+
+ private S add(final S event) {
+ S newValue;
+ S value;
+ do {
+ value = ref.get();
+ newValue = entityAccess.aggregate(value, event);
+ } while (!ref.compareAndSet(value, newValue));
+
+ return newValue;
+ }
+
+ public S getValue() {
+ return ref.get();
+ }
+
+ public boolean isExpired() {
+ // entityAccess.getTimestamp(curValue) represents the time at which the current value
+ // was last updated. If the last value is less than current time - 1 binPrecision, then it
+ // means that we've rolled over and need to reset the value.
+ final long maxExpectedTimePeriod = System.currentTimeMillis() - TimeUnit.MILLISECONDS.convert(numConfiguredBins, binPrecision);
+
+ final S curValue = ref.get();
+ return (entityAccess.getTimestamp(curValue) < maxExpectedTimePeriod);
+ }
+
+ public S addOrReset(final S event) {
+ // entityAccess.getTimestamp(curValue) represents the time at which the current value
+ // was last updated. If the last value is less than current time - 1 binPrecision, then it
+ // means that we've rolled over and need to reset the value.
+ final long maxExpectedTimePeriod = System.currentTimeMillis() - TimeUnit.MILLISECONDS.convert(1, binPrecision);
+
+ final S curValue = ref.get();
+ if (entityAccess.getTimestamp(curValue) < maxExpectedTimePeriod) {
+ ref.compareAndSet(curValue, entityAccess.createNew());
+ }
+ return add(event);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/timebuffer/TimestampedLong.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/timebuffer/TimestampedLong.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/timebuffer/TimestampedLong.java
new file mode 100644
index 0000000..07d31ea
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/timebuffer/TimestampedLong.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.timebuffer;
+
+public class TimestampedLong {
+
+ private final Long value;
+ private final long timestamp = System.currentTimeMillis();
+
+ public TimestampedLong(final Long value) {
+ this.value = value;
+ }
+
+ public Long getValue() {
+ return value;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/remote/io/TestCompressionInputOutputStreams.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/remote/io/TestCompressionInputOutputStreams.java b/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/remote/io/TestCompressionInputOutputStreams.java
new file mode 100644
index 0000000..bd30a96
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/remote/io/TestCompressionInputOutputStreams.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote.io;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
+
+import org.apache.nifi.remote.io.CompressionInputStream;
+import org.apache.nifi.remote.io.CompressionOutputStream;
+
+import org.junit.Test;
+
+public class TestCompressionInputOutputStreams {
+
+ @Test
+ public void testSimple() throws IOException {
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ final byte[] data = "Hello, World!".getBytes("UTF-8");
+
+ final CompressionOutputStream cos = new CompressionOutputStream(baos);
+ cos.write(data);
+ cos.flush();
+ cos.close();
+
+ final byte[] compressedBytes = baos.toByteArray();
+ final CompressionInputStream cis = new CompressionInputStream(new ByteArrayInputStream(compressedBytes));
+ final byte[] decompressed = readFully(cis);
+
+ assertTrue(Arrays.equals(data, decompressed));
+ }
+
+ @Test
+ public void testDataLargerThanBuffer() throws IOException {
+ final String str = "The quick brown fox jumps over the lazy dog\r\n\n\n\r";
+
+ final StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < 100; i++) {
+ sb.append(str);
+ }
+ final byte[] data = sb.toString().getBytes("UTF-8");
+
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ final CompressionOutputStream cos = new CompressionOutputStream(baos, 8192);
+ cos.write(data);
+ cos.flush();
+ cos.close();
+
+ final byte[] compressedBytes = baos.toByteArray();
+ final CompressionInputStream cis = new CompressionInputStream(new ByteArrayInputStream(compressedBytes));
+ final byte[] decompressed = readFully(cis);
+
+ assertTrue(Arrays.equals(data, decompressed));
+ }
+
+ @Test
+ public void testDataLargerThanBufferWhileFlushing() throws IOException {
+ final String str = "The quick brown fox jumps over the lazy dog\r\n\n\n\r";
+ final byte[] data = str.getBytes("UTF-8");
+
+ final StringBuilder sb = new StringBuilder();
+ final byte[] data1024;
+
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ final CompressionOutputStream cos = new CompressionOutputStream(baos, 8192);
+ for (int i = 0; i < 1024; i++) {
+ cos.write(data);
+ cos.flush();
+ sb.append(str);
+ }
+ cos.close();
+ data1024 = sb.toString().getBytes("UTF-8");
+
+ final byte[] compressedBytes = baos.toByteArray();
+ final CompressionInputStream cis = new CompressionInputStream(new ByteArrayInputStream(compressedBytes));
+ final byte[] decompressed = readFully(cis);
+
+ assertTrue(Arrays.equals(data1024, decompressed));
+ }
+
+ @Test
+ public void testSendingMultipleFilesBackToBackOnSameStream() throws IOException {
+ final String str = "The quick brown fox jumps over the lazy dog\r\n\n\n\r";
+ final byte[] data = str.getBytes("UTF-8");
+
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ final CompressionOutputStream cos = new CompressionOutputStream(baos, 8192);
+ for (int i = 0; i < 512; i++) {
+ cos.write(data);
+ cos.flush();
+ }
+ cos.close();
+
+ final CompressionOutputStream cos2 = new CompressionOutputStream(baos, 8192);
+ for (int i = 0; i < 512; i++) {
+ cos2.write(data);
+ cos2.flush();
+ }
+ cos2.close();
+
+ final byte[] data512;
+ final StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < 512; i++) {
+ sb.append(str);
+ }
+ data512 = sb.toString().getBytes("UTF-8");
+
+ final byte[] compressedBytes = baos.toByteArray();
+ final ByteArrayInputStream bais = new ByteArrayInputStream(compressedBytes);
+
+ final CompressionInputStream cis = new CompressionInputStream(bais);
+ final byte[] decompressed = readFully(cis);
+ assertTrue(Arrays.equals(data512, decompressed));
+
+ final CompressionInputStream cis2 = new CompressionInputStream(bais);
+ final byte[] decompressed2 = readFully(cis2);
+ assertTrue(Arrays.equals(data512, decompressed2));
+ }
+
+ private byte[] readFully(final InputStream in) throws IOException {
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ final byte[] buffer = new byte[65536];
+ int len;
+ while ((len = in.read(buffer)) >= 0) {
+ baos.write(buffer, 0, len);
+ }
+
+ return baos.toByteArray();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/stream/io/ByteCountingInputStreamTest.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/stream/io/ByteCountingInputStreamTest.java b/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/stream/io/ByteCountingInputStreamTest.java
new file mode 100644
index 0000000..27b1493
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/stream/io/ByteCountingInputStreamTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io;
+
+import junit.framework.TestCase;
+
+public class ByteCountingInputStreamTest extends TestCase {
+
+ final ByteArrayInputStream reader = new ByteArrayInputStream("abcdefghijklmnopqrstuvwxyz".getBytes());
+
+ public void testReset() throws Exception {
+
+ final ByteArrayInputStream reader = new ByteArrayInputStream("abcdefghijklmnopqrstuvwxyz".getBytes());
+ final ByteCountingInputStream bcis = new ByteCountingInputStream(reader);
+ int tmp;
+
+ /* verify first 2 bytes */
+ tmp = bcis.read();
+ assertEquals(tmp, 97);
+ tmp = bcis.read();
+ assertEquals(tmp, 98);
+
+ /* save bytes read and place mark */
+ final long bytesAtMark = bcis.getBytesRead();
+ bcis.mark(0);
+
+ /* verify next 2 bytes */
+ tmp = bcis.read();
+ assertEquals(tmp, 99);
+ tmp = bcis.read();
+ assertEquals(tmp, 100);
+
+ /* verify reset returns to position when mark was placed */
+ bcis.reset();
+ assertEquals(bytesAtMark, bcis.getBytesRead());
+
+ /* verify that the reset bug has been fixed (bug would reduce bytes read count) */
+ bcis.reset();
+ assertEquals(bytesAtMark, bcis.getBytesRead());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/stream/io/TestLeakyBucketThrottler.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/stream/io/TestLeakyBucketThrottler.java b/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/stream/io/TestLeakyBucketThrottler.java
new file mode 100644
index 0000000..52bd8de
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/stream/io/TestLeakyBucketThrottler.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.stream.io;
+
+import org.apache.nifi.stream.io.ByteArrayInputStream;
+import org.apache.nifi.stream.io.ByteArrayOutputStream;
+import org.apache.nifi.stream.io.LeakyBucketStreamThrottler;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+@Ignore("Tests are time-based")
+public class TestLeakyBucketThrottler {
+
+ @Test(timeout = 10000)
+ public void testOutputStreamInterface() throws IOException {
+ // throttle rate at 1 MB/sec
+ final LeakyBucketStreamThrottler throttler = new LeakyBucketStreamThrottler(1024 * 1024);
+
+ final byte[] data = new byte[1024 * 1024 * 4];
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ final OutputStream throttledOut = throttler.newThrottledOutputStream(baos);
+
+ final long start = System.currentTimeMillis();
+ throttledOut.write(data);
+ throttler.close();
+ final long millis = System.currentTimeMillis() - start;
+ // should take 4 sec give or take
+ assertTrue(millis > 3000);
+ assertTrue(millis < 6000);
+ }
+
+ @Test(timeout = 10000)
+ public void testInputStreamInterface() throws IOException {
+ // throttle rate at 1 MB/sec
+ final LeakyBucketStreamThrottler throttler = new LeakyBucketStreamThrottler(1024 * 1024);
+
+ final byte[] data = new byte[1024 * 1024 * 4];
+ final ByteArrayInputStream bais = new ByteArrayInputStream(data);
+ final InputStream throttledIn = throttler.newThrottledInputStream(bais);
+
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ final byte[] buffer = new byte[4096];
+ final long start = System.currentTimeMillis();
+ int len;
+ while ((len = throttledIn.read(buffer)) > 0) {
+ baos.write(buffer, 0, len);
+ }
+ throttler.close();
+ final long millis = System.currentTimeMillis() - start;
+ // should take 4 sec give or take
+ assertTrue(millis > 3000);
+ assertTrue(millis < 6000);
+ baos.close();
+ }
+
+ @Test(timeout = 10000)
+ public void testDirectInterface() throws IOException, InterruptedException {
+ // throttle rate at 1 MB/sec
+ final LeakyBucketStreamThrottler throttler = new LeakyBucketStreamThrottler(1024 * 1024);
+
+ // create 3 threads, each sending ~2 MB
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ final List<Thread> threads = new ArrayList<Thread>();
+ for (int i = 0; i < 3; i++) {
+ final Thread t = new WriterThread(i, throttler, baos);
+ threads.add(t);
+ }
+
+ final long start = System.currentTimeMillis();
+ for (final Thread t : threads) {
+ t.start();
+ }
+
+ for (final Thread t : threads) {
+ t.join();
+ }
+ final long elapsed = System.currentTimeMillis() - start;
+
+ throttler.close();
+
+ // To send 15 MB, it should have taken at least 5 seconds and no more than 7 seconds, to
+ // allow for busy-ness and the fact that we could write a tiny bit more than the limit.
+ assertTrue(elapsed > 5000);
+ assertTrue(elapsed < 7000);
+
+ // ensure bytes were copied out appropriately
+ assertEquals(3 * (2 * 1024 * 1024 + 1), baos.getBufferLength());
+ assertEquals((byte) 'A', baos.getUnderlyingBuffer()[baos.getBufferLength() - 1]);
+ }
+
+ private static class WriterThread extends Thread {
+
+ private final int idx;
+ private final byte[] data = new byte[1024 * 1024 * 2 + 1];
+ private final LeakyBucketStreamThrottler throttler;
+ private final OutputStream out;
+
+ public WriterThread(final int idx, final LeakyBucketStreamThrottler throttler, final OutputStream out) {
+ this.idx = idx;
+ this.throttler = throttler;
+ this.out = out;
+ this.data[this.data.length - 1] = (byte) 'A';
+ }
+
+ @Override
+ public void run() {
+ long startMillis = System.currentTimeMillis();
+ long bytesWritten = 0L;
+ try {
+ throttler.copy(new ByteArrayInputStream(data), out);
+ } catch (IOException e) {
+ e.printStackTrace();
+ return;
+ }
+ long now = System.currentTimeMillis();
+ long millisElapsed = now - startMillis;
+ bytesWritten += data.length;
+ float bytesPerSec = (float) bytesWritten / (float) millisElapsed * 1000F;
+ System.out.println(idx + " : copied data at a rate of " + bytesPerSec + " bytes/sec");
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/TestNaiveSearchRingBuffer.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/TestNaiveSearchRingBuffer.java b/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/TestNaiveSearchRingBuffer.java
new file mode 100644
index 0000000..0838e96
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/TestNaiveSearchRingBuffer.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+
+public class TestNaiveSearchRingBuffer {
+
+ @Test
+ public void testAddAndCompare() {
+ final byte[] pattern = new byte[]{
+ '\r', '0', 38, 48
+ };
+
+ final byte[] search = new byte[]{
+ '\r', '0', 38, 58, 58, 83, 78, '\r', '0', 38, 48, 83, 92, 78, 4, 38
+ };
+
+ final NaiveSearchRingBuffer circ = new NaiveSearchRingBuffer(pattern);
+ int counter = -1;
+ for (final byte b : search) {
+ counter++;
+ final boolean matched = circ.addAndCompare(b);
+ if (counter == 10) {
+ assertTrue(matched);
+ } else {
+ assertFalse(matched);
+ }
+ }
+ }
+
+ @Test
+ public void testGetOldestByte() {
+ final byte[] pattern = new byte[]{
+ '\r', '0', 38, 48
+ };
+
+ final byte[] search = new byte[]{
+ '\r', '0', 38, 58, 58, 83, 78, (byte) 223, (byte) 227, (byte) 250, '\r', '0', 38, 48, 83, 92, 78, 4, 38
+ };
+
+ final NaiveSearchRingBuffer circ = new NaiveSearchRingBuffer(pattern);
+ int counter = -1;
+ for (final byte b : search) {
+ counter++;
+ final boolean matched = circ.addAndCompare(b);
+ if (counter == 13) {
+ assertTrue(matched);
+ } else {
+ assertFalse(matched);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/file/monitor/TestCompoundUpdateMonitor.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/file/monitor/TestCompoundUpdateMonitor.java b/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/file/monitor/TestCompoundUpdateMonitor.java
new file mode 100644
index 0000000..ec04efb
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/file/monitor/TestCompoundUpdateMonitor.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.file.monitor;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.file.Path;
+import java.util.UUID;
+
+import org.junit.Test;
+
+public class TestCompoundUpdateMonitor {
+
+ @Test
+ public void test() throws IOException {
+ final UpdateMonitor lastModified = new LastModifiedMonitor();
+ final MD5SumMonitor md5 = new MD5SumMonitor();
+ final CompoundUpdateMonitor compound = new CompoundUpdateMonitor(lastModified, md5);
+
+ final File file = new File("target/" + UUID.randomUUID().toString());
+ if (file.exists()) {
+ assertTrue(file.delete());
+ }
+ assertTrue(file.createNewFile());
+
+ final Path path = file.toPath();
+
+ final Object curState = compound.getCurrentState(path);
+ final Object state2 = compound.getCurrentState(path);
+
+ assertEquals(curState, state2);
+ file.setLastModified(System.currentTimeMillis() + 1000L);
+ final Object state3 = compound.getCurrentState(path);
+ assertEquals(state2, state3);
+
+ final Object state4 = compound.getCurrentState(path);
+ assertEquals(state3, state4);
+
+ final long lastModifiedDate = file.lastModified();
+ try (final OutputStream out = new FileOutputStream(file)) {
+ out.write("Hello".getBytes("UTF-8"));
+ }
+
+ file.setLastModified(lastModifiedDate);
+
+ final Object state5 = compound.getCurrentState(path);
+ assertNotSame(state4, state5);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/file/monitor/TestSynchronousFileWatcher.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/file/monitor/TestSynchronousFileWatcher.java b/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/file/monitor/TestSynchronousFileWatcher.java
new file mode 100644
index 0000000..3440c16
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/file/monitor/TestSynchronousFileWatcher.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.file.monitor;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+
+import org.junit.Test;
+
+public class TestSynchronousFileWatcher {
+
+ @Test
+ public void testIt() throws UnsupportedEncodingException, IOException, InterruptedException {
+ final Path path = Paths.get("target/1.txt");
+ Files.copy(new ByteArrayInputStream("Hello, World!".getBytes("UTF-8")), path, StandardCopyOption.REPLACE_EXISTING);
+ final UpdateMonitor monitor = new MD5SumMonitor();
+
+ final SynchronousFileWatcher watcher = new SynchronousFileWatcher(path, monitor, 10L);
+ assertFalse(watcher.checkAndReset());
+ Thread.sleep(30L);
+ assertFalse(watcher.checkAndReset());
+
+ final FileOutputStream fos = new FileOutputStream(path.toFile());
+ try {
+ fos.write("Good-bye, World!".getBytes("UTF-8"));
+ fos.getFD().sync();
+ } finally {
+ fos.close();
+ }
+
+ assertTrue(watcher.checkAndReset());
+ assertFalse(watcher.checkAndReset());
+
+ Thread.sleep(30L);
+ assertFalse(watcher.checkAndReset());
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/timebuffer/TestRingBuffer.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/timebuffer/TestRingBuffer.java b/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/timebuffer/TestRingBuffer.java
new file mode 100644
index 0000000..b01b495
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/timebuffer/TestRingBuffer.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.timebuffer;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.nifi.util.RingBuffer;
+import org.apache.nifi.util.RingBuffer.ForEachEvaluator;
+import org.apache.nifi.util.RingBuffer.IterationDirection;
+
+import org.junit.Test;
+
+/**
+ *
+ */
+public class TestRingBuffer {
+
+ @Test
+ public void testGetNewestElement() {
+ final RingBuffer<Integer> ringBuffer = new RingBuffer<>(10);
+
+ for (int i = 0; i < 11; i++) {
+ ringBuffer.add(i);
+ assertEquals(i, ringBuffer.getNewestElement().intValue());
+ }
+ }
+
+ @Test
+ public void testAsList() {
+ final RingBuffer<Integer> ringBuffer = new RingBuffer<>(10);
+
+ final List<Integer> emptyList = ringBuffer.asList();
+ assertTrue(emptyList.isEmpty());
+
+ for (int i = 0; i < 3; i++) {
+ ringBuffer.add(i);
+ }
+
+ List<Integer> list = ringBuffer.asList();
+ assertEquals(3, list.size());
+ for (int i = 0; i < 3; i++) {
+ assertEquals(Integer.valueOf(i), list.get(i));
+ }
+
+ for (int i = 3; i < 10; i++) {
+ ringBuffer.add(i);
+ }
+
+ list = ringBuffer.asList();
+ assertEquals(10, list.size());
+ for (int i = 0; i < 10; i++) {
+ assertEquals(Integer.valueOf(i), list.get(i));
+ }
+ }
+
+ @Test
+ public void testIterateForwards() {
+ final RingBuffer<Integer> ringBuffer = new RingBuffer<>(10);
+
+ final int[] values = new int[]{3, 5, 20, 7};
+ for (final int v : values) {
+ ringBuffer.add(v);
+ }
+
+ final AtomicInteger countHolder = new AtomicInteger(0);
+ ringBuffer.forEach(new ForEachEvaluator<Integer>() {
+ int counter = 0;
+
+ @Override
+ public boolean evaluate(final Integer value) {
+ final int expected = values[counter++];
+ countHolder.incrementAndGet();
+ assertEquals(expected, value.intValue());
+ return true;
+ }
+
+ }, IterationDirection.FORWARD);
+
+ assertEquals(4, countHolder.get());
+ }
+
+ @Test
+ public void testIterateForwardsAfterFull() {
+ final RingBuffer<Integer> ringBuffer = new RingBuffer<>(10);
+
+ for (int i = 0; i < 12; i++) {
+ ringBuffer.add(i);
+ }
+
+ final int[] values = new int[]{3, 5, 20, 7};
+ for (final int v : values) {
+ ringBuffer.add(v);
+ }
+
+ ringBuffer.forEach(new ForEachEvaluator<Integer>() {
+ int counter = 0;
+
+ @Override
+ public boolean evaluate(final Integer value) {
+ if (counter < 6) {
+ assertEquals(counter + 6, value.intValue());
+ } else {
+ final int expected = values[counter - 6];
+ assertEquals(expected, value.intValue());
+ }
+
+ counter++;
+ return true;
+ }
+
+ }, IterationDirection.FORWARD);
+ }
+
+ @Test
+ public void testIterateBackwards() {
+ final RingBuffer<Integer> ringBuffer = new RingBuffer<>(10);
+
+ final int[] values = new int[]{3, 5, 20, 7};
+ for (final int v : values) {
+ ringBuffer.add(v);
+ }
+
+ final AtomicInteger countHolder = new AtomicInteger(0);
+ ringBuffer.forEach(new ForEachEvaluator<Integer>() {
+ int counter = 0;
+
+ @Override
+ public boolean evaluate(final Integer value) {
+ final int index = values.length - 1 - counter;
+ final int expected = values[index];
+ countHolder.incrementAndGet();
+
+ assertEquals(expected, value.intValue());
+ counter++;
+ return true;
+ }
+
+ }, IterationDirection.BACKWARD);
+
+ assertEquals(4, countHolder.get());
+ }
+
+ @Test
+ public void testIterateBackwardsAfterFull() {
+ final RingBuffer<Integer> ringBuffer = new RingBuffer<>(10);
+
+ for (int i = 0; i < 12; i++) {
+ ringBuffer.add(i);
+ }
+
+ final int[] values = new int[]{3, 5, 20, 7};
+ for (final int v : values) {
+ ringBuffer.add(v);
+ }
+
+ ringBuffer.forEach(new ForEachEvaluator<Integer>() {
+ int counter = 0;
+
+ @Override
+ public boolean evaluate(final Integer value) {
+ if (counter < values.length) {
+ final int index = values.length - 1 - counter;
+ final int expected = values[index];
+
+ assertEquals(expected, value.intValue());
+ counter++;
+ }
+
+ return true;
+ }
+
+ }, IterationDirection.BACKWARD);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/timebuffer/TestTimedBuffer.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/timebuffer/TestTimedBuffer.java b/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/timebuffer/TestTimedBuffer.java
new file mode 100644
index 0000000..39ca330
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/timebuffer/TestTimedBuffer.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.timebuffer;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Test;
+
+public class TestTimedBuffer {
+
+ @Test
+ public void testAgesOff() throws InterruptedException {
+ final LongEntityAccess access = new LongEntityAccess();
+ final TimedBuffer<TimestampedLong> buffer = new TimedBuffer<>(TimeUnit.SECONDS, 2, access);
+
+ buffer.add(new TimestampedLong(1000000L));
+ TimestampedLong aggregate = buffer.getAggregateValue(System.currentTimeMillis() - 30000L);
+ assertEquals(1000000L, aggregate.getValue().longValue());
+ Thread.sleep(1000L);
+ aggregate = buffer.getAggregateValue(System.currentTimeMillis() - 30000L);
+ assertEquals(1000000L, aggregate.getValue().longValue());
+ Thread.sleep(1500L);
+ aggregate = buffer.getAggregateValue(System.currentTimeMillis() - 30000L);
+ assertNull(aggregate);
+ }
+
+ @Test
+ public void testAggregation() throws InterruptedException {
+ final LongEntityAccess access = new LongEntityAccess();
+ final TimedBuffer<TimestampedLong> buffer = new TimedBuffer<>(TimeUnit.SECONDS, 2, access);
+
+ buffer.add(new TimestampedLong(1000000L));
+ buffer.add(new TimestampedLong(1000000L));
+ buffer.add(new TimestampedLong(25000L));
+
+ TimestampedLong aggregate = buffer.getAggregateValue(System.currentTimeMillis() - 30000L);
+ assertEquals(2025000L, aggregate.getValue().longValue());
+ Thread.sleep(1000L);
+ aggregate = buffer.getAggregateValue(System.currentTimeMillis() - 30000L);
+ assertEquals(2025000L, aggregate.getValue().longValue());
+ Thread.sleep(1500L);
+ aggregate = buffer.getAggregateValue(System.currentTimeMillis() - 30000L);
+ assertNull(aggregate);
+ }
+
+ private static class TimestampedLong {
+
+ private final Long value;
+ private final long timestamp = System.currentTimeMillis();
+
+ public TimestampedLong(final Long value) {
+ this.value = value;
+ }
+
+ public Long getValue() {
+ return value;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+ }
+
+ private static class LongEntityAccess implements EntityAccess<TimestampedLong> {
+
+ @Override
+ public TimestampedLong aggregate(TimestampedLong oldValue, TimestampedLong toAdd) {
+ if (oldValue == null && toAdd == null) {
+ return new TimestampedLong(0L);
+ } else if (oldValue == null) {
+ return toAdd;
+ } else if (toAdd == null) {
+ return oldValue;
+ }
+
+ return new TimestampedLong(oldValue.getValue().longValue() + toAdd.getValue().longValue());
+ }
+
+ @Override
+ public TimestampedLong createNew() {
+ return new TimestampedLong(0L);
+ }
+
+ @Override
+ public long getTimestamp(TimestampedLong entity) {
+ return entity == null ? 0L : entity.getTimestamp();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-utils/src/test/resources/logback-test.xml
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/test/resources/logback-test.xml b/nifi-commons/nifi-utils/src/test/resources/logback-test.xml
new file mode 100644
index 0000000..8651d47
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/test/resources/logback-test.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration scan="true" scanPeriod="30 seconds">
+ <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+ <pattern>%-4r [%t] %-5p %c - %m%n</pattern>
+ </encoder>
+ </appender>
+
+ <!-- valid logging levels: TRACE, DEBUG, INFO, WARN, ERROR -->
+ <logger name="org.apache.nifi" level="DEBUG"/>
+
+ <root level="INFO">
+ <appender-ref ref="CONSOLE"/>
+ </root>
+
+</configuration>
+
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-web-utils/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-web-utils/pom.xml b/nifi-commons/nifi-web-utils/pom.xml
new file mode 100644
index 0000000..8c51d7b
--- /dev/null
+++ b/nifi-commons/nifi-web-utils/pom.xml
@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-commons</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>nifi-web-utils</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-security-utils</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-json</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-web-utils/src/main/java/org/apache/nifi/web/util/ClientUtils.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-web-utils/src/main/java/org/apache/nifi/web/util/ClientUtils.java b/nifi-commons/nifi-web-utils/src/main/java/org/apache/nifi/web/util/ClientUtils.java
new file mode 100644
index 0000000..1eaf366
--- /dev/null
+++ b/nifi-commons/nifi-web-utils/src/main/java/org/apache/nifi/web/util/ClientUtils.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.util;
+
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientHandlerException;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.UniformInterfaceException;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.core.util.MultivaluedMapImpl;
+import java.net.URI;
+import java.util.Map;
+import javax.ws.rs.core.MediaType;
+
+/**
+ *
+ */
+public class ClientUtils {
+
+ private final Client client;
+
+ public ClientUtils(Client client) {
+ this.client = client;
+ }
+
+ /**
+ * Gets the content at the specified URI.
+ *
+ * @param uri the URI to get the content of
+ * @return the client response resulting from getting the content of the URI
+ * @throws ClientHandlerException if issues occur handling the request
+ * @throws UniformInterfaceException if any interface violations occur
+ */
+ public ClientResponse get(final URI uri) throws ClientHandlerException, UniformInterfaceException {
+ return get(uri, null);
+ }
+
+ /**
+ * Gets the content at the specified URI using the given query parameters.
+ *
+ * @param uri the URI to get the content of
+ * @param queryParams the query parameters to use in the request
+ * @return the client response resulting from getting the content of the URI
+ * @throws ClientHandlerException if issues occur handling the request
+ * @throws UniformInterfaceException if any interface violations occur
+ */
+ public ClientResponse get(final URI uri, final Map<String, String> queryParams) throws ClientHandlerException, UniformInterfaceException {
+ // perform the request
+ WebResource webResource = client.resource(uri);
+ if (queryParams != null) {
+ for (final Map.Entry<String, String> queryEntry : queryParams.entrySet()) {
+ webResource = webResource.queryParam(queryEntry.getKey(), queryEntry.getValue());
+ }
+ }
+
+ return webResource.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ }
+
+ /**
+ * Performs a POST using the specified url and entity body.
+ *
+ * @param uri the URI to post to
+ * @param entity the item to post
+ * @return the client response of the request
+ */
+ public ClientResponse post(URI uri, Object entity) throws ClientHandlerException, UniformInterfaceException {
+ // get the resource
+ WebResource.Builder resourceBuilder = client.resource(uri).accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON);
+
+ // include the request entity
+ if (entity != null) {
+ resourceBuilder = resourceBuilder.entity(entity);
+ }
+
+ // perform the request
+ return resourceBuilder.post(ClientResponse.class);
+ }
+
+ /**
+ * Performs a POST using the specified url and form data.
+ *
+ * @param uri the uri to post to
+ * @param formData the data to post
+ * @return the client reponse of the post
+ */
+ public ClientResponse post(URI uri, Map<String, String> formData) throws ClientHandlerException, UniformInterfaceException {
+ // convert the form data
+ MultivaluedMapImpl entity = new MultivaluedMapImpl();
+ for (String key : formData.keySet()) {
+ entity.add(key, formData.get(key));
+ }
+
+ // get the resource
+ WebResource.Builder resourceBuilder = client.resource(uri).accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_FORM_URLENCODED);
+
+ // add the form data if necessary
+ if (!entity.isEmpty()) {
+ resourceBuilder = resourceBuilder.entity(entity);
+ }
+
+ // perform the request
+ return resourceBuilder.post(ClientResponse.class);
+ }
+
+ /**
+ * Performs a HEAD request to the specified URI.
+ *
+ * @param uri the uri to request the head of
+ * @return the client response of the request
+ * @throws ClientHandlerException for issues handling the request
+ * @throws UniformInterfaceException for issues with the request
+ */
+ public ClientResponse head(final URI uri) throws ClientHandlerException, UniformInterfaceException {
+ // perform the request
+ WebResource webResource = client.resource(uri);
+ return webResource.head();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-web-utils/src/main/java/org/apache/nifi/web/util/ObjectMapperResolver.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-web-utils/src/main/java/org/apache/nifi/web/util/ObjectMapperResolver.java b/nifi-commons/nifi-web-utils/src/main/java/org/apache/nifi/web/util/ObjectMapperResolver.java
new file mode 100644
index 0000000..4e7f5b6
--- /dev/null
+++ b/nifi-commons/nifi-web-utils/src/main/java/org/apache/nifi/web/util/ObjectMapperResolver.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.util;
+
+import javax.ws.rs.ext.ContextResolver;
+import javax.ws.rs.ext.Provider;
+import org.codehaus.jackson.map.AnnotationIntrospector;
+import org.codehaus.jackson.map.DeserializationConfig;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.SerializationConfig;
+import org.codehaus.jackson.map.annotate.JsonSerialize.Inclusion;
+import org.codehaus.jackson.xc.JaxbAnnotationIntrospector;
+
+@Provider
+public class ObjectMapperResolver implements ContextResolver<ObjectMapper> {
+
+ private final ObjectMapper mapper;
+
+ public ObjectMapperResolver() throws Exception {
+ mapper = new ObjectMapper();
+
+ final AnnotationIntrospector jaxbIntrospector = new JaxbAnnotationIntrospector();
+ final SerializationConfig serializationConfig = mapper.getSerializationConfig();
+ final DeserializationConfig deserializationConfig = mapper.getDeserializationConfig();
+
+ mapper.setSerializationConfig(serializationConfig.withSerializationInclusion(Inclusion.NON_NULL).withAnnotationIntrospector(jaxbIntrospector));
+ mapper.setDeserializationConfig(deserializationConfig.withAnnotationIntrospector(jaxbIntrospector));
+ }
+
+ @Override
+ public ObjectMapper getContext(Class<?> objectType) {
+ return mapper;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-web-utils/src/main/java/org/apache/nifi/web/util/WebUtils.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-web-utils/src/main/java/org/apache/nifi/web/util/WebUtils.java b/nifi-commons/nifi-web-utils/src/main/java/org/apache/nifi/web/util/WebUtils.java
new file mode 100644
index 0000000..e27f91c
--- /dev/null
+++ b/nifi-commons/nifi-web-utils/src/main/java/org/apache/nifi/web/util/WebUtils.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.util;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateParsingException;
+import java.security.cert.X509Certificate;
+import java.util.List;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLPeerUnverifiedException;
+import javax.net.ssl.SSLSession;
+
+import org.apache.nifi.security.util.CertificateUtils;
+
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.config.ClientConfig;
+import com.sun.jersey.api.client.config.DefaultClientConfig;
+import com.sun.jersey.api.json.JSONConfiguration;
+import com.sun.jersey.client.urlconnection.HTTPSProperties;
+
+/**
+ * Common utilities related to web development.
+ *
+ */
+public final class WebUtils {
+
+ private static Logger logger = LoggerFactory.getLogger(WebUtils.class);
+
+ final static ReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private WebUtils() {
+ }
+
+ /**
+ * Creates a client for non-secure requests. The client will be created
+ * using the given configuration. Additionally, the client will be
+ * automatically configured for JSON serialization/deserialization.
+ *
+ * @param config client configuration
+ *
+ * @return a Client instance
+ */
+ public static Client createClient(final ClientConfig config) {
+ return createClientHelper(config, null);
+ }
+
+ /**
+ * Creates a client for secure requests. The client will be created using
+ * the given configuration and security context. Additionally, the client
+ * will be automatically configured for JSON serialization/deserialization.
+ *
+ * @param config client configuration
+ * @param ctx security context
+ *
+ * @return a Client instance
+ */
+ public static Client createClient(final ClientConfig config, final SSLContext ctx) {
+ return createClientHelper(config, ctx);
+ }
+
+ /**
+ * A helper method for creating clients. The client will be created using
+ * the given configuration and security context. Additionally, the client
+ * will be automatically configured for JSON serialization/deserialization.
+ *
+ * @param config client configuration
+ * @param ctx security context, which may be null for non-secure client
+ * creation
+ *
+ * @return a Client instance
+ */
+ private static Client createClientHelper(final ClientConfig config, final SSLContext ctx) {
+
+ final ClientConfig finalConfig = (config == null) ? new DefaultClientConfig() : config;
+
+ if (ctx != null && StringUtils.isBlank((String) finalConfig.getProperty(HTTPSProperties.PROPERTY_HTTPS_PROPERTIES))) {
+
+ // custom hostname verifier that checks subject alternative names against the hostname of the URI
+ final HostnameVerifier hostnameVerifier = new HostnameVerifier() {
+ @Override
+ public boolean verify(final String hostname, final SSLSession ssls) {
+
+ try {
+ for (final Certificate peerCertificate : ssls.getPeerCertificates()) {
+ if (peerCertificate instanceof X509Certificate) {
+ final X509Certificate x509Cert = (X509Certificate) peerCertificate;
+ final List<String> subjectAltNames = CertificateUtils.getSubjectAlternativeNames(x509Cert);
+ if (subjectAltNames.contains(hostname.toLowerCase())) {
+ return true;
+ }
+ }
+ }
+ } catch (final SSLPeerUnverifiedException | CertificateParsingException ex) {
+ logger.warn("Hostname Verification encountered exception verifying hostname due to: " + ex, ex);
+ }
+
+ return false;
+ }
+ };
+
+ finalConfig.getProperties().put(HTTPSProperties.PROPERTY_HTTPS_PROPERTIES, new HTTPSProperties(hostnameVerifier, ctx));
+ }
+
+ finalConfig.getFeatures().put(JSONConfiguration.FEATURE_POJO_MAPPING, Boolean.TRUE);
+ finalConfig.getClasses().add(ObjectMapperResolver.class);
+
+ // web client for restful request
+ return Client.create(finalConfig);
+
+ }
+
+ /**
+ * Serializes the given object to hexadecimal. Serialization uses Java's
+ * native serialization mechanism, the ObjectOutputStream.
+ *
+ * @param obj an object
+ * @return the serialized object as hex
+ */
+ public static String serializeObjectToHex(final Serializable obj) {
+
+ final ByteArrayOutputStream serializedObj = new ByteArrayOutputStream();
+
+ // IOException can never be thrown because we are serializing to an in memory byte array
+ try {
+ final ObjectOutputStream oos = new ObjectOutputStream(serializedObj);
+ oos.writeObject(obj);
+ oos.close();
+ } catch (final IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+
+ logger.debug(String.format("Serialized object '%s' size: %d", obj, serializedObj.size()));
+
+ // hex encode the binary
+ return new String(Hex.encodeHex(serializedObj.toByteArray(), /* tolowercase */ true));
+ }
+
+ /**
+ * Deserializes a Java serialized, hex-encoded string into a Java object.
+ * This method is the inverse of the serializeObjectToHex method in this
+ * class.
+ *
+ * @param hexEncodedObject a string
+ * @return the object
+ * @throws ClassNotFoundException if the class could not be found
+ */
+ public static Serializable deserializeHexToObject(final String hexEncodedObject) throws ClassNotFoundException {
+
+ // decode the hex encoded object
+ byte[] serializedObj;
+ try {
+ serializedObj = Hex.decodeHex(hexEncodedObject.toCharArray());
+ } catch (final DecoderException de) {
+ throw new IllegalArgumentException(de);
+ }
+
+ // IOException can never be thrown because we are deserializing from an in memory byte array
+ try {
+ // deserialize bytes into object
+ ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(serializedObj));
+ return (Serializable) ois.readObject();
+ } catch (final IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-write-ahead-log/.gitignore
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-write-ahead-log/.gitignore b/nifi-commons/nifi-write-ahead-log/.gitignore
new file mode 100755
index 0000000..19f2e00
--- /dev/null
+++ b/nifi-commons/nifi-write-ahead-log/.gitignore
@@ -0,0 +1,2 @@
+/target
+/target
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-write-ahead-log/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-write-ahead-log/pom.xml b/nifi-commons/nifi-write-ahead-log/pom.xml
new file mode 100644
index 0000000..7e514a7
--- /dev/null
+++ b/nifi-commons/nifi-write-ahead-log/pom.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-commons</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>nifi-write-ahead-log</artifactId>
+ <packaging>jar</packaging>
+ <dependencies>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-utils</artifactId>
+ </dependency>
+ </dependencies>
+</project>
[41/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-assembly/README.md
----------------------------------------------------------------------
diff --git a/nifi-assembly/README.md b/nifi-assembly/README.md
new file mode 100644
index 0000000..4fb4d3b
--- /dev/null
+++ b/nifi-assembly/README.md
@@ -0,0 +1,110 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+# Apache NiFi
+
+Apache NiFi is an easy to use, powerful, and reliable system to process and distribute data.
+
+## Table of Contents
+
+- [Features](#features)
+- [Getting Started](#getting-started)
+- [Getting Help](#getting-help)
+- [Requirements](#requirements)
+- [License](#license)
+- [Export Control] (#export-control)
+
+## Features
+
+Apache NiFi was made for dataflow. It supports highly configurable directed graphs of data routing, transformation, and system mediation logic. Some of its key features include:
+
+- Web-based user interface
+ - Seamless experience for design, control, and monitoring
+- Highly configurable
+ - Loss tolerant vs guaranteed delivery
+ - Low latency vs high throughput
+ - Dynamic prioritization
+ - Flows can be modified at runtime
+ - Back pressure
+- Data Provenance
+ - Track dataflow from beginning to end
+- Designed for extension
+ - Build your own processors and more
+ - Enables rapid development and effective testing
+- Secure
+ - SSL, SSH, HTTPS, encrypted content, etc...
+ - Pluggable role-based authentication/authorization
+
+## Getting Started
+
+To start NiFi:
+- [linux/osx] execute bin/nifi.sh start
+- [windows] execute bin/start-nifi.bat
+- Direct your browser to http://localhost:8080/nifi/
+
+## Getting Help
+If you have questions, you can reach out to our mailing list: dev@nifi.apache.org
+([archive](http://mail-archives.apache.org/mod_mbox/nifi-dev)).
+We're also often available in IRC: #nifi on
+[irc.freenode.net](http://webchat.freenode.net/?channels=#nifi).
+
+## Requirements
+* JDK 1.7 or higher
+
+## License
+
+Except as otherwise noted this software is licensed under the
+[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0.html)
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+## Export Control
+
+This distribution includes cryptographic software. The country in which you
+currently reside may have restrictions on the import, possession, use, and/or
+re-export to another country, of encryption software. BEFORE using any
+encryption software, please check your country's laws, regulations and
+policies concerning the import, possession, or use, and re-export of encryption
+software, to see if this is permitted. See <http://www.wassenaar.org/> for more
+information.
+
+The U.S. Government Department of Commerce, Bureau of Industry and Security
+(BIS), has classified this software as Export Commodity Control Number (ECCN)
+5D002.C.1, which includes information security software using or performing
+cryptographic functions with asymmetric algorithms. The form and manner of this
+Apache Software Foundation distribution makes it eligible for export under the
+License Exception ENC Technology Software Unrestricted (TSU) exception (see the
+BIS Export Administration Regulations, Section 740.13) for both object code and
+source code.
+
+The following provides more details on the included cryptographic software:
+
+Apache NiFi uses BouncyCastle, Jasypt, JCraft Inc., and the built-in
+java cryptography libraries for SSL, SSH, and the protection
+of sensitive configuration parameters. See
+http://bouncycastle.org/about.html
+http://www.jasypt.org/faq.html
+http://jcraft.com/c-info.html
+http://www.oracle.com/us/products/export/export-regulations-345813.html
+for more details on each of these libraries cryptography features.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-assembly/pom.xml b/nifi-assembly/pom.xml
new file mode 100644
index 0000000..60a6545
--- /dev/null
+++ b/nifi-assembly/pom.xml
@@ -0,0 +1,501 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+license agreements. See the NOTICE file distributed with this work for additional
+information regarding copyright ownership. The ASF licenses this file to
+You under the Apache License, Version 2.0 (the "License"); you may not use
+this file except in compliance with the License. You may obtain a copy of
+the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+by applicable law or agreed to in writing, software distributed under the
+License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+OF ANY KIND, either express or implied. See the License for the specific
+language governing permissions and limitations under the License. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>nifi-assembly</artifactId>
+ <packaging>pom</packaging>
+ <description>This is the assembly Apache NiFi</description>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <finalName>nifi-${project.version}</finalName>
+ <attach>false</attach>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make shared resource</id>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <phase>package</phase>
+ <configuration>
+ <archiverConfig>
+ <defaultDirectoryMode>0775</defaultDirectoryMode>
+ <directoryMode>0775</directoryMode>
+ <fileMode>0664</fileMode>
+ </archiverConfig>
+ <descriptors>
+ <descriptor>src/main/assembly/dependencies.xml</descriptor>
+ </descriptors>
+ <tarLongFileMode>posix</tarLongFileMode>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>jcl-over-slf4j</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>jul-to-slf4j</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>log4j-over-slf4j</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-runtime</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-bootstrap</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-resources</artifactId>
+ <classifier>resources</classifier>
+ <scope>runtime</scope>
+ <type>zip</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-docs</artifactId>
+ <classifier>resources</classifier>
+ <scope>runtime</scope>
+ <type>zip</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-framework-nar</artifactId>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-provenance-repository-nar</artifactId>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-standard-services-api-nar</artifactId>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-ssl-context-service-nar</artifactId>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-distributed-cache-services-nar</artifactId>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-standard-nar</artifactId>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-jetty-bundle</artifactId>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-update-attribute-nar</artifactId>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-hadoop-libraries-nar</artifactId>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-hadoop-nar</artifactId>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kafka-nar</artifactId>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-http-context-map-nar</artifactId>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kite-nar</artifactId>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-flume-nar</artifactId>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-dbcp-service-nar</artifactId>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-mongodb-nar</artifactId>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-solr-nar</artifactId>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-social-media-nar</artifactId>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-hl7-nar</artifactId>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-language-translation-nar</artifactId>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-geo-nar</artifactId>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-aws-nar</artifactId>
+ <type>nar</type>
+ </dependency>
+ </dependencies>
+
+ <properties>
+ <!--Wrapper Properties -->
+ <nifi.wrapper.jvm.heap.initial.mb>256</nifi.wrapper.jvm.heap.initial.mb>
+ <nifi.wrapper.jvm.heap.max.mb>512</nifi.wrapper.jvm.heap.max.mb>
+ <nifi.initial.permgen.size.mb>128</nifi.initial.permgen.size.mb>
+ <nifi.max.permgen.size.mb>128</nifi.max.permgen.size.mb>
+ <nifi.wrapper.logfile.maxsize>10m</nifi.wrapper.logfile.maxsize>
+ <nifi.wrapper.logfile.maxfiles>10</nifi.wrapper.logfile.maxfiles>
+
+ <!-- nifi.properties: core properties -->
+ <nifi.version>${project.version}</nifi.version>
+ <nifi.flowcontroller.autoResumeState>true</nifi.flowcontroller.autoResumeState>
+ <nifi.flowcontroller.graceful.shutdown.period>10 sec</nifi.flowcontroller.graceful.shutdown.period>
+ <nifi.flowservice.writedelay.interval>500 ms</nifi.flowservice.writedelay.interval>
+ <nifi.administrative.yield.duration>30 sec</nifi.administrative.yield.duration>
+ <nifi.bored.yield.duration>10 millis</nifi.bored.yield.duration>
+
+ <nifi.flow.configuration.file>./conf/flow.xml.gz</nifi.flow.configuration.file>
+ <nifi.flow.configuration.archive.dir>./conf/archive/</nifi.flow.configuration.archive.dir>
+ <nifi.authority.provider.configuration.file>./conf/authority-providers.xml</nifi.authority.provider.configuration.file>
+ <nifi.templates.directory>./conf/templates</nifi.templates.directory>
+ <nifi.database.directory>./database_repository</nifi.database.directory>
+
+ <nifi.flowfile.repository.implementation>org.apache.nifi.controller.repository.WriteAheadFlowFileRepository</nifi.flowfile.repository.implementation>
+ <nifi.flowfile.repository.directory>./flowfile_repository</nifi.flowfile.repository.directory>
+ <nifi.flowfile.repository.partitions>256</nifi.flowfile.repository.partitions>
+ <nifi.flowfile.repository.checkpoint.interval>2 mins</nifi.flowfile.repository.checkpoint.interval>
+ <nifi.flowfile.repository.always.sync>false</nifi.flowfile.repository.always.sync>
+ <nifi.swap.manager.implementation>org.apache.nifi.controller.FileSystemSwapManager</nifi.swap.manager.implementation>
+ <nifi.queue.swap.threshold>20000</nifi.queue.swap.threshold>
+ <nifi.swap.in.period>5 sec</nifi.swap.in.period>
+ <nifi.swap.in.threads>1</nifi.swap.in.threads>
+ <nifi.swap.out.period>5 sec</nifi.swap.out.period>
+ <nifi.swap.out.threads>4</nifi.swap.out.threads>
+
+ <nifi.content.repository.implementation>org.apache.nifi.controller.repository.FileSystemRepository</nifi.content.repository.implementation>
+ <nifi.content.claim.max.appendable.size>10 MB</nifi.content.claim.max.appendable.size>
+ <nifi.content.claim.max.flow.files>100</nifi.content.claim.max.flow.files>
+ <nifi.content.repository.directory.default>./content_repository</nifi.content.repository.directory.default>
+ <nifi.content.repository.archive.max.retention.period>12 hours</nifi.content.repository.archive.max.retention.period>
+ <nifi.content.repository.archive.max.usage.percentage>50%</nifi.content.repository.archive.max.usage.percentage>
+ <nifi.content.repository.archive.enabled>true</nifi.content.repository.archive.enabled>
+ <nifi.content.repository.always.sync>false</nifi.content.repository.always.sync>
+ <nifi.content.viewer.url>/nifi-content-viewer/</nifi.content.viewer.url>
+
+ <nifi.restore.directory />
+ <nifi.ui.banner.text />
+ <nifi.ui.autorefresh.interval>30 sec</nifi.ui.autorefresh.interval>
+ <nifi.nar.library.directory>./lib</nifi.nar.library.directory>
+ <nifi.nar.working.directory>./work/nar/</nifi.nar.working.directory>
+ <nifi.documentation.working.directory>./work/docs/components</nifi.documentation.working.directory>
+
+ <nifi.sensitive.props.algorithm>PBEWITHMD5AND256BITAES-CBC-OPENSSL</nifi.sensitive.props.algorithm>
+ <nifi.sensitive.props.provider>BC</nifi.sensitive.props.provider>
+ <nifi.h2.url.append>;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE</nifi.h2.url.append>
+
+ <nifi.remote.input.socket.port>9990</nifi.remote.input.socket.port>
+
+ <!-- persistent provenance repository properties -->
+ <nifi.provenance.repository.implementation>org.apache.nifi.provenance.PersistentProvenanceRepository</nifi.provenance.repository.implementation>
+ <nifi.provenance.repository.directory.default>./provenance_repository</nifi.provenance.repository.directory.default>
+ <nifi.provenance.repository.max.storage.time>24 hours</nifi.provenance.repository.max.storage.time>
+ <nifi.provenance.repository.max.storage.size>1 GB</nifi.provenance.repository.max.storage.size>
+ <nifi.provenance.repository.rollover.time>30 secs</nifi.provenance.repository.rollover.time>
+ <nifi.provenance.repository.rollover.size>100 MB</nifi.provenance.repository.rollover.size>
+ <nifi.provenance.repository.query.threads>2</nifi.provenance.repository.query.threads>
+ <nifi.provenance.repository.compress.on.rollover>true</nifi.provenance.repository.compress.on.rollover>
+ <nifi.provenance.repository.indexed.fields>EventType, FlowFileUUID, Filename, ProcessorID, Relationship</nifi.provenance.repository.indexed.fields>
+ <nifi.provenance.repository.indexed.attributes />
+ <nifi.provenance.repository.index.shard.size>500 MB</nifi.provenance.repository.index.shard.size>
+ <nifi.provenance.repository.always.sync>false</nifi.provenance.repository.always.sync>
+ <nifi.provenance.repository.journal.count>16</nifi.provenance.repository.journal.count>
+ <nifi.provenance.repository.max.attribute.length>65536</nifi.provenance.repository.max.attribute.length>
+
+ <!-- volatile provenance repository properties -->
+ <nifi.provenance.repository.buffer.size>100000</nifi.provenance.repository.buffer.size>
+
+ <!-- Component status repository properties -->
+ <nifi.components.status.repository.implementation>org.apache.nifi.controller.status.history.VolatileComponentStatusRepository</nifi.components.status.repository.implementation>
+ <nifi.components.status.repository.buffer.size>1440</nifi.components.status.repository.buffer.size>
+ <nifi.components.status.snapshot.frequency>1 min</nifi.components.status.snapshot.frequency>
+
+ <!-- nifi.properties: web properties -->
+ <nifi.web.war.directory>./lib</nifi.web.war.directory>
+ <nifi.web.http.host />
+ <nifi.web.http.port>8080</nifi.web.http.port>
+ <nifi.web.https.host />
+ <nifi.web.https.port />
+ <nifi.jetty.work.dir>./work/jetty</nifi.jetty.work.dir>
+ <nifi.web.jetty.threads>200</nifi.web.jetty.threads>
+
+ <!-- nifi.properties: security properties -->
+ <nifi.security.keystore />
+ <nifi.security.keystoreType />
+ <nifi.security.keystorePasswd />
+ <nifi.security.keyPasswd />
+ <nifi.security.truststore />
+ <nifi.security.truststoreType />
+ <nifi.security.truststorePasswd />
+ <nifi.security.needClientAuth />
+ <nifi.security.authorizedUsers.file>./conf/authorized-users.xml</nifi.security.authorizedUsers.file>
+ <nifi.security.user.credential.cache.duration>24 hours</nifi.security.user.credential.cache.duration>
+ <nifi.security.user.authority.provider>file-provider</nifi.security.user.authority.provider>
+ <nifi.security.x509.principal.extractor />
+ <nifi.security.support.new.account.requests />
+ <nifi.security.ocsp.responder.url />
+ <nifi.security.ocsp.responder.certificate />
+
+ <!-- nifi.properties: cluster common properties (cluster manager and nodes must have same values) -->
+ <nifi.cluster.protocol.heartbeat.interval>5 sec</nifi.cluster.protocol.heartbeat.interval>
+ <nifi.cluster.protocol.is.secure>false</nifi.cluster.protocol.is.secure>
+ <nifi.cluster.protocol.socket.timeout>30 sec</nifi.cluster.protocol.socket.timeout>
+ <nifi.cluster.protocol.connection.handshake.timeout>45 sec</nifi.cluster.protocol.connection.handshake.timeout>
+ <nifi.cluster.protocol.use.multicast>false</nifi.cluster.protocol.use.multicast>
+ <nifi.cluster.protocol.multicast.address />
+ <nifi.cluster.protocol.multicast.port />
+ <nifi.cluster.protocol.multicast.service.broadcast.delay>500 ms</nifi.cluster.protocol.multicast.service.broadcast.delay>
+ <nifi.cluster.protocol.multicast.service.locator.attempts>3</nifi.cluster.protocol.multicast.service.locator.attempts>
+ <nifi.cluster.protocol.multicast.service.locator.attempts.delay>1 sec</nifi.cluster.protocol.multicast.service.locator.attempts.delay>
+
+ <!-- nifi.properties: cluster node properties (only configure for cluster nodes) -->
+ <nifi.cluster.is.node>false</nifi.cluster.is.node>
+ <nifi.cluster.node.address />
+ <nifi.cluster.node.protocol.port />
+ <nifi.cluster.node.protocol.threads>2</nifi.cluster.node.protocol.threads>
+ <nifi.cluster.node.unicast.manager.address />
+ <nifi.cluster.node.unicast.manager.protocol.port />
+
+ <!-- nifi.properties: cluster manager properties (only configure for cluster manager) -->
+ <nifi.cluster.is.manager>false</nifi.cluster.is.manager>
+ <nifi.cluster.manager.address />
+ <nifi.cluster.manager.protocol.port />
+ <nifi.cluster.manager.node.firewall.file />
+ <nifi.cluster.manager.node.event.history.size>10</nifi.cluster.manager.node.event.history.size>
+ <nifi.cluster.manager.node.api.connection.timeout>30 sec</nifi.cluster.manager.node.api.connection.timeout>
+ <nifi.cluster.manager.node.api.read.timeout>30 sec</nifi.cluster.manager.node.api.read.timeout>
+ <nifi.cluster.manager.node.api.request.threads>10</nifi.cluster.manager.node.api.request.threads>
+ <nifi.cluster.manager.flow.retrieval.delay>5 sec</nifi.cluster.manager.flow.retrieval.delay>
+ <nifi.cluster.manager.protocol.threads>10</nifi.cluster.manager.protocol.threads>
+ <nifi.cluster.manager.safemode.duration>0 sec</nifi.cluster.manager.safemode.duration>
+ </properties>
+ <profiles>
+ <profile>
+ <id>rpm</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>unpack-shared-resources</id>
+ <goals>
+ <goal>unpack-dependencies</goal>
+ </goals>
+ <phase>generate-resources</phase>
+ <configuration>
+ <outputDirectory>${project.build.directory}/generated-resources</outputDirectory>
+ <includeArtifactIds>nifi-resources</includeArtifactIds>
+ <includeGroupIds>org.apache.nifi</includeGroupIds>
+ <excludeTransitive>false</excludeTransitive>
+ </configuration>
+ </execution>
+ <execution>
+ <id>unpack-docs</id>
+ <goals>
+ <goal>unpack-dependencies</goal>
+ </goals>
+ <phase>generate-resources</phase>
+ <configuration>
+ <outputDirectory>${project.build.directory}/generated-docs</outputDirectory>
+ <includeArtifactIds>nifi-docs</includeArtifactIds>
+ <includeGroupIds>org.apache.nifi</includeGroupIds>
+ <excludeTransitive>false</excludeTransitive>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>rpm-maven-plugin</artifactId>
+ <configuration>
+ <summary>Apache NiFi</summary>
+ <description>Apache NiFi is dataflow system based on the Flow-Based Programming concepts.</description>
+ <license>Apache License, Version 2.0 and others (see included LICENSE file)</license>
+ <url>http://nifi.apache.org</url>
+ <group>Utilities</group>
+ <prefix>/opt/nifi</prefix>
+ <defineStatements>
+ <defineStatement>_use_internal_dependency_generator 0</defineStatement>
+ </defineStatements>
+ <defaultDirmode>750</defaultDirmode>
+ <defaultFilemode>640</defaultFilemode>
+ <defaultUsername>root</defaultUsername>
+ <defaultGroupname>root</defaultGroupname>
+ </configuration>
+ <executions>
+ <execution>
+ <id>build-bin-rpm</id>
+ <goals>
+ <goal>attached-rpm</goal>
+ </goals>
+ <configuration>
+ <classifier>bin</classifier>
+ <provides>
+ <provide>nifi</provide>
+ </provides>
+ <mappings>
+ <mapping>
+ <directory>/opt/nifi/nifi-${project.version}</directory>
+ </mapping>
+ <mapping>
+ <directory>/opt/nifi/nifi-${project.version}</directory>
+ <sources>
+ <source>
+ <location>./LICENSE</location>
+ </source>
+ <source>
+ <location>./NOTICE</location>
+ </source>
+ <source>
+ <location>./README.md</location>
+ <destination>README</destination>
+ </source>
+ </sources>
+ </mapping>
+ <mapping>
+ <directory>/opt/nifi/nifi-${project.version}/bin</directory>
+ <filemode>750</filemode>
+ <sources>
+ <source>
+ <location>${project.build.directory}/generated-resources/bin/nifi.sh</location>
+ <destination>nifi.sh</destination>
+ <filter>true</filter>
+ </source>
+ </sources>
+ </mapping>
+ <mapping>
+ <directory>/opt/nifi/nifi-${project.version}/conf</directory>
+ <configuration>true</configuration>
+ <sources>
+ <source>
+ <location>${project.build.directory}/generated-resources/conf</location>
+ <filter>true</filter>
+ </source>
+ </sources>
+ </mapping>
+ <mapping>
+ <directory>/opt/nifi/nifi-${project.version}/lib</directory>
+ <dependency>
+ <excludes>
+ <exclude>org.apache.nifi:nifi-bootstrap</exclude>
+ <exclude>org.apache.nifi:nifi-resources</exclude>
+ <exclude>org.apache.nifi:nifi-docs</exclude>
+ </excludes>
+ </dependency>
+ </mapping>
+ <mapping>
+ <directory>/opt/nifi/nifi-${project.version}/lib/bootstrap</directory>
+ <dependency>
+ <includes>
+ <include>org.apache.nifi:nifi-bootstrap</include>
+ </includes>
+ </dependency>
+ </mapping>
+ <mapping>
+ <directory>/opt/nifi/nifi-${project.version}/docs</directory>
+ <sources>
+ <source>
+ <location>${project.build.directory}/generated-docs</location>
+ </source>
+ </sources>
+ </mapping>
+ </mappings>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-assembly/src/main/assembly/dependencies.xml
----------------------------------------------------------------------
diff --git a/nifi-assembly/src/main/assembly/dependencies.xml b/nifi-assembly/src/main/assembly/dependencies.xml
new file mode 100644
index 0000000..0e5362c
--- /dev/null
+++ b/nifi-assembly/src/main/assembly/dependencies.xml
@@ -0,0 +1,140 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<assembly>
+ <id>bin</id>
+ <formats>
+ <format>dir</format>
+ <format>zip</format>
+ <format>tar.gz</format>
+ </formats>
+ <includeBaseDirectory>true</includeBaseDirectory>
+ <baseDirectory>nifi-${project.version}</baseDirectory>
+
+ <dependencySets>
+ <!-- Write out all dependency artifacts to lib directory -->
+ <dependencySet>
+ <scope>runtime</scope>
+ <useProjectArtifact>false</useProjectArtifact>
+ <outputDirectory>lib</outputDirectory>
+ <directoryMode>0770</directoryMode>
+ <fileMode>0660</fileMode>
+ <useTransitiveFiltering>true</useTransitiveFiltering>
+ <excludes>
+ <exclude>nifi-bootstrap</exclude>
+ <exclude>nifi-resources</exclude>
+ <exclude>nifi-docs</exclude>
+ </excludes>
+ </dependencySet>
+
+ <!-- Write out the bootstrap lib component to its own dir -->
+ <dependencySet>
+ <scope>runtime</scope>
+ <useProjectArtifact>false</useProjectArtifact>
+ <outputDirectory>lib/bootstrap</outputDirectory>
+ <directoryMode>0770</directoryMode>
+ <fileMode>0660</fileMode>
+ <useTransitiveFiltering>true</useTransitiveFiltering>
+ <includes>
+ <include>nifi-bootstrap</include>
+ <include>slf4j-api</include>
+ <include>logback-classic</include>
+ </includes>
+ </dependencySet>
+
+ <!-- Write out the conf directory contents -->
+ <dependencySet>
+ <scope>runtime</scope>
+ <useProjectArtifact>false</useProjectArtifact>
+ <outputDirectory>./</outputDirectory>
+ <directoryMode>0770</directoryMode>
+ <fileMode>0664</fileMode>
+ <useTransitiveFiltering>true</useTransitiveFiltering>
+ <includes>
+ <include>nifi-resources</include>
+ </includes>
+ <unpack>true</unpack>
+ <unpackOptions>
+ <filtered>true</filtered>
+ <includes>
+ <include>conf/*</include>
+ </includes>
+ </unpackOptions>
+ </dependencySet>
+
+ <!-- Write out the bin directory contents -->
+ <dependencySet>
+ <scope>runtime</scope>
+ <useProjectArtifact>false</useProjectArtifact>
+ <outputDirectory>./</outputDirectory>
+ <directoryMode>0770</directoryMode>
+ <fileMode>0770</fileMode>
+ <useTransitiveFiltering>true</useTransitiveFiltering>
+ <includes>
+ <include>nifi-resources</include>
+ </includes>
+ <unpack>true</unpack>
+ <unpackOptions>
+ <filtered>true</filtered>
+ <includes>
+ <include>bin/*</include>
+ </includes>
+ </unpackOptions>
+ </dependencySet>
+
+ <!-- Writes out the docs directory contents -->
+ <dependencySet>
+ <scope>runtime</scope>
+ <useProjectArtifact>false</useProjectArtifact>
+ <outputDirectory>docs/</outputDirectory>
+ <useTransitiveFiltering>true</useTransitiveFiltering>
+ <includes>
+ <include>nifi-docs</include>
+ </includes>
+ <unpack>true</unpack>
+ <unpackOptions>
+ <filtered>false</filtered>
+ <excludes>
+ <!-- LICENSE and NOTICE both covered by top-level -->
+ <exclude>LICENSE</exclude>
+ <exclude>NOTICE</exclude>
+ </excludes>
+ </unpackOptions>
+ </dependencySet>
+ </dependencySets>
+ <files>
+ <file>
+ <source>./README.md</source>
+ <outputDirectory>./</outputDirectory>
+ <destName>README</destName>
+ <fileMode>0644</fileMode>
+ <filtered>true</filtered>
+ </file>
+ <file>
+ <source>./LICENSE</source>
+ <outputDirectory>./</outputDirectory>
+ <destName>LICENSE</destName>
+ <fileMode>0644</fileMode>
+ <filtered>true</filtered>
+ </file>
+ <file>
+ <source>./NOTICE</source>
+ <outputDirectory>./</outputDirectory>
+ <destName>NOTICE</destName>
+ <fileMode>0644</fileMode>
+ <filtered>true</filtered>
+ </file>
+ </files>
+</assembly>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-bootstrap/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-bootstrap/pom.xml b/nifi-bootstrap/pom.xml
new file mode 100644
index 0000000..2975a5a
--- /dev/null
+++ b/nifi-bootstrap/pom.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>nifi-bootstrap</artifactId>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/BootstrapCodec.java
----------------------------------------------------------------------
diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/BootstrapCodec.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/BootstrapCodec.java
new file mode 100644
index 0000000..3817875
--- /dev/null
+++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/BootstrapCodec.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.bootstrap;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.util.Arrays;
+
+import org.apache.nifi.bootstrap.exception.InvalidCommandException;
+
+public class BootstrapCodec {
+
+ private final RunNiFi runner;
+ private final BufferedReader reader;
+ private final BufferedWriter writer;
+
+ public BootstrapCodec(final RunNiFi runner, final InputStream in, final OutputStream out) {
+ this.runner = runner;
+ this.reader = new BufferedReader(new InputStreamReader(in));
+ this.writer = new BufferedWriter(new OutputStreamWriter(out));
+ }
+
+ public void communicate() throws IOException {
+ final String line = reader.readLine();
+ final String[] splits = line.split(" ");
+ if (splits.length < 0) {
+ throw new IOException("Received invalid command from NiFi: " + line);
+ }
+
+ final String cmd = splits[0];
+ final String[] args;
+ if (splits.length == 1) {
+ args = new String[0];
+ } else {
+ args = Arrays.copyOfRange(splits, 1, splits.length);
+ }
+
+ try {
+ processRequest(cmd, args);
+ } catch (final InvalidCommandException ice) {
+ throw new IOException("Received invalid command from NiFi: " + line + " : " + ice.getMessage() == null ? "" : "Details: " + ice.toString());
+ }
+ }
+
+ private void processRequest(final String cmd, final String[] args) throws InvalidCommandException, IOException {
+ switch (cmd) {
+ case "PORT": {
+ if (args.length != 2) {
+ throw new InvalidCommandException();
+ }
+
+ final int port;
+ try {
+ port = Integer.parseInt(args[0]);
+ } catch (final NumberFormatException nfe) {
+ throw new InvalidCommandException("Invalid Port number; should be integer between 1 and 65535");
+ }
+
+ if (port < 1 || port > 65535) {
+ throw new InvalidCommandException("Invalid Port number; should be integer between 1 and 65535");
+ }
+
+ final String secretKey = args[1];
+
+ runner.setNiFiCommandControlPort(port, secretKey);
+ writer.write("OK");
+ writer.newLine();
+ writer.flush();
+ }
+ break;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/NiFiListener.java
----------------------------------------------------------------------
diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/NiFiListener.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/NiFiListener.java
new file mode 100644
index 0000000..8d74f16
--- /dev/null
+++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/NiFiListener.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.bootstrap;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.nifi.bootstrap.util.LimitingInputStream;
+
+public class NiFiListener {
+
+ private ServerSocket serverSocket;
+ private volatile Listener listener;
+
+ int start(final RunNiFi runner) throws IOException {
+ serverSocket = new ServerSocket();
+ serverSocket.bind(new InetSocketAddress("localhost", 0));
+
+ final int localPort = serverSocket.getLocalPort();
+ listener = new Listener(serverSocket, runner);
+ final Thread listenThread = new Thread(listener);
+ listenThread.setName("Listen to NiFi");
+ listenThread.setDaemon(true);
+ listenThread.start();
+ return localPort;
+ }
+
+ public void stop() throws IOException {
+ final Listener listener = this.listener;
+ if (listener == null) {
+ return;
+ }
+
+ listener.stop();
+ }
+
+ private class Listener implements Runnable {
+
+ private final ServerSocket serverSocket;
+ private final ExecutorService executor;
+ private final RunNiFi runner;
+ private volatile boolean stopped = false;
+
+ public Listener(final ServerSocket serverSocket, final RunNiFi runner) {
+ this.serverSocket = serverSocket;
+ this.executor = Executors.newFixedThreadPool(2, new ThreadFactory() {
+ @Override
+ public Thread newThread(final Runnable runnable) {
+ final Thread t = Executors.defaultThreadFactory().newThread(runnable);
+ t.setDaemon(true);
+ t.setName("NiFi Bootstrap Command Listener");
+ return t;
+ }
+ });
+
+ this.runner = runner;
+ }
+
+ public void stop() throws IOException {
+ stopped = true;
+
+ executor.shutdown();
+ try {
+ executor.awaitTermination(3, TimeUnit.SECONDS);
+ } catch (final InterruptedException ie) {
+ }
+
+ serverSocket.close();
+ }
+
+ @Override
+ public void run() {
+ while (!serverSocket.isClosed()) {
+ try {
+ if (stopped) {
+ return;
+ }
+
+ final Socket socket;
+ try {
+ socket = serverSocket.accept();
+ } catch (final IOException ioe) {
+ if (stopped) {
+ return;
+ }
+
+ throw ioe;
+ }
+
+ executor.submit(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ // we want to ensure that we don't try to read data from an InputStream directly
+ // by a BufferedReader because any user on the system could open a socket and send
+ // a multi-gigabyte file without any new lines in order to crash the Bootstrap,
+ // which in turn may cause the Shutdown Hook to shutdown NiFi.
+ // So we will limit the amount of data to read to 4 KB
+ final InputStream limitingIn = new LimitingInputStream(socket.getInputStream(), 4096);
+ final BootstrapCodec codec = new BootstrapCodec(runner, limitingIn, socket.getOutputStream());
+ codec.communicate();
+ } catch (final Throwable t) {
+ System.out.println("Failed to communicate with NiFi due to " + t);
+ t.printStackTrace();
+ } finally {
+ try {
+ socket.close();
+ } catch (final IOException ioe) {
+ }
+ }
+ }
+ });
+ } catch (final Throwable t) {
+ System.err.println("Failed to receive information from NiFi due to " + t);
+ t.printStackTrace();
+ }
+ }
+ }
+ }
+}
[04/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/GetPreviousValues.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/GetPreviousValues.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/GetPreviousValues.java
new file mode 100644
index 0000000..569439b
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/GetPreviousValues.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import java.util.List;
+import java.util.Map;
+import org.apache.nifi.admin.dao.ActionDAO;
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.authorization.AuthorityProvider;
+import org.apache.nifi.history.PreviousValue;
+
+/**
+ * Gets the action with the specified id.
+ */
+public class GetPreviousValues implements AdministrationAction<Map<String, List<PreviousValue>>> {
+
+ private final String componentId;
+
+ public GetPreviousValues(String componentId) {
+ this.componentId = componentId;
+ }
+
+ @Override
+ public Map<String, List<PreviousValue>> execute(DAOFactory daoFactory, AuthorityProvider authorityProvider) {
+ ActionDAO actionDao = daoFactory.getActionDAO();
+ return actionDao.getPreviousValues(componentId);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/GetUserGroupAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/GetUserGroupAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/GetUserGroupAction.java
new file mode 100644
index 0000000..5377c46
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/GetUserGroupAction.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.admin.dao.DataAccessException;
+import org.apache.nifi.admin.dao.UserDAO;
+import org.apache.nifi.authorization.AuthorityProvider;
+import org.apache.nifi.user.NiFiUserGroup;
+
+/**
+ *
+ */
+public class GetUserGroupAction implements AdministrationAction<NiFiUserGroup> {
+
+ private final String group;
+
+ public GetUserGroupAction(String group) {
+ this.group = group;
+ }
+
+ @Override
+ public NiFiUserGroup execute(DAOFactory daoFactory, AuthorityProvider authorityProvider) throws DataAccessException {
+ final UserDAO userDAO = daoFactory.getUserDAO();
+ final NiFiUserGroup userGroup = new NiFiUserGroup();
+
+ // set the group
+ userGroup.setGroup(group);
+
+ // get the users in this group
+ userGroup.setUsers(userDAO.findUsersForGroup(group));
+
+ // return the group
+ return userGroup;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/GetUsersAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/GetUsersAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/GetUsersAction.java
new file mode 100644
index 0000000..42d180e
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/GetUsersAction.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import java.util.Collection;
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.admin.dao.DataAccessException;
+import org.apache.nifi.admin.dao.UserDAO;
+import org.apache.nifi.authorization.AuthorityProvider;
+import org.apache.nifi.user.NiFiUser;
+
+/**
+ *
+ */
+public class GetUsersAction implements AdministrationAction<Collection<NiFiUser>> {
+
+ @Override
+ public Collection<NiFiUser> execute(DAOFactory daoFactory, AuthorityProvider authorityProvider) throws DataAccessException {
+ // get a UserDAO
+ UserDAO userDAO = daoFactory.getUserDAO();
+
+ // return the desired user
+ return userDAO.findUsers();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/HasPendingUserAccounts.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/HasPendingUserAccounts.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/HasPendingUserAccounts.java
new file mode 100644
index 0000000..3325642
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/HasPendingUserAccounts.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.admin.dao.DataAccessException;
+import org.apache.nifi.admin.dao.UserDAO;
+import org.apache.nifi.authorization.AuthorityProvider;
+
+/**
+ * Action for creating a NiFiUser account.
+ */
+public class HasPendingUserAccounts extends AbstractUserAction<Boolean> {
+
+ @Override
+ public Boolean execute(DAOFactory daoFactory, AuthorityProvider authorityProvider) throws DataAccessException {
+ UserDAO userDao = daoFactory.getUserDAO();
+ return userDao.hasPendingUserAccounts();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/InvalidateUserAccountAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/InvalidateUserAccountAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/InvalidateUserAccountAction.java
new file mode 100644
index 0000000..14596b2
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/InvalidateUserAccountAction.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.admin.dao.DataAccessException;
+import org.apache.nifi.admin.dao.UserDAO;
+import org.apache.nifi.admin.service.AccountNotFoundException;
+import org.apache.nifi.authorization.AuthorityProvider;
+import org.apache.nifi.user.NiFiUser;
+
+/**
+ * Invalidates a user account.
+ */
+public class InvalidateUserAccountAction implements AdministrationAction<Void> {
+
+ private final String id;
+
+ public InvalidateUserAccountAction(String id) {
+ this.id = id;
+ }
+
+ @Override
+ public Void execute(DAOFactory daoFactory, AuthorityProvider authorityProvider) throws DataAccessException {
+ UserDAO userDao = daoFactory.getUserDAO();
+
+ // get the current user details
+ NiFiUser user = userDao.findUserById(id);
+
+ // ensure the user exists
+ if (user == null) {
+ throw new AccountNotFoundException(String.format("Unable to find account with ID %s.", id));
+ }
+
+ // invalidate the user account
+ user.setLastVerified(null);
+
+ // create the user entry
+ userDao.updateUser(user);
+
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/InvalidateUserGroupAccountsAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/InvalidateUserGroupAccountsAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/InvalidateUserGroupAccountsAction.java
new file mode 100644
index 0000000..0cb7e14
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/InvalidateUserGroupAccountsAction.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.admin.dao.DataAccessException;
+import org.apache.nifi.admin.dao.UserDAO;
+import org.apache.nifi.authorization.AuthorityProvider;
+
+/**
+ * Invalidates a user account.
+ */
+public class InvalidateUserGroupAccountsAction implements AdministrationAction<Void> {
+
+ private final String group;
+
+ public InvalidateUserGroupAccountsAction(String group) {
+ this.group = group;
+ }
+
+ @Override
+ public Void execute(DAOFactory daoFactory, AuthorityProvider authorityProvider) throws DataAccessException {
+ UserDAO userDao = daoFactory.getUserDAO();
+
+ // create the user entry
+ userDao.updateGroupVerification(group, null);
+
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/PurgeActionsAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/PurgeActionsAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/PurgeActionsAction.java
new file mode 100644
index 0000000..b5a2883
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/PurgeActionsAction.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import java.util.Date;
+import org.apache.nifi.action.Action;
+import org.apache.nifi.admin.dao.ActionDAO;
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.authorization.AuthorityProvider;
+
+/**
+ * Purges actions up to a specified end date.
+ */
+public class PurgeActionsAction implements AdministrationAction<Void> {
+
+ private final Date end;
+ private final Action purgeAction;
+
+ public PurgeActionsAction(Date end, Action purgeAction) {
+ this.end = end;
+ this.purgeAction = purgeAction;
+ }
+
+ @Override
+ public Void execute(DAOFactory daoFactory, AuthorityProvider authorityProvider) {
+ ActionDAO actionDao = daoFactory.getActionDAO();
+
+ // remove the corresponding actions
+ actionDao.deleteActions(end);
+
+ // create a purge action
+ actionDao.createAction(purgeAction);
+
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/RequestUserAccountAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/RequestUserAccountAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/RequestUserAccountAction.java
new file mode 100644
index 0000000..3dce6d9
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/RequestUserAccountAction.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import java.util.Date;
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.admin.dao.DataAccessException;
+import org.apache.nifi.admin.dao.UserDAO;
+import org.apache.nifi.authorization.AuthorityProvider;
+import org.apache.nifi.security.util.CertificateUtils;
+import org.apache.nifi.user.AccountStatus;
+import org.apache.nifi.user.NiFiUser;
+
+/**
+ *
+ */
+public class RequestUserAccountAction implements AdministrationAction<NiFiUser> {
+
+ private final String dn;
+ private final String justification;
+
+ public RequestUserAccountAction(String dn, String justification) {
+ this.dn = dn;
+ this.justification = justification;
+ }
+
+ @Override
+ public NiFiUser execute(DAOFactory daoFactory, AuthorityProvider authorityProvider) throws DataAccessException {
+ UserDAO userDao = daoFactory.getUserDAO();
+
+ // determine if this user already exists
+ NiFiUser user = userDao.findUserByDn(dn);
+ if (user != null) {
+ throw new IllegalArgumentException(String.format("User account for %s already exists.", dn));
+ }
+
+ // create the user
+ user = new NiFiUser();
+ user.setDn(dn);
+ user.setUserName(CertificateUtils.extractUsername(dn));
+ user.setJustification(justification);
+ user.setStatus(AccountStatus.PENDING);
+
+ // update user timestamps
+ Date now = new Date();
+ user.setCreation(now);
+
+ // create the new user account
+ userDao.createUser(user);
+
+ return user;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/SeedUserAccountsAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/SeedUserAccountsAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/SeedUserAccountsAction.java
new file mode 100644
index 0000000..6665ac0
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/SeedUserAccountsAction.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import java.util.HashSet;
+import java.util.Set;
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.admin.dao.DataAccessException;
+import org.apache.nifi.admin.dao.UserDAO;
+import org.apache.nifi.admin.service.AdministrationException;
+import org.apache.nifi.authorization.Authority;
+import org.apache.nifi.authorization.AuthorityProvider;
+import org.apache.nifi.authorization.exception.AuthorityAccessException;
+import org.apache.nifi.authorization.exception.UnknownIdentityException;
+import org.apache.nifi.security.util.CertificateUtils;
+import org.apache.nifi.user.AccountStatus;
+import org.apache.nifi.user.NiFiUser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Seeds the user accounts. This action is performed at start up because it
+ * takes the users specified in the authority provider and makes them available
+ * to be seen in the UI. This happens because the UI loads the users from the
+ * cache. Without pre loading the users, the table in the UI would only show a
+ * given user once they have visited the application.
+ */
+public class SeedUserAccountsAction extends AbstractUserAction<Void> {
+
+ private static final Logger logger = LoggerFactory.getLogger(SeedUserAccountsAction.class);
+
+ @Override
+ public Void execute(DAOFactory daoFactory, AuthorityProvider authorityProvider) throws DataAccessException {
+ UserDAO userDao = daoFactory.getUserDAO();
+ Set<String> authorizedDns = new HashSet<>();
+
+ // get the current user cache
+ final Set<NiFiUser> existingUsers;
+ try {
+ existingUsers = userDao.findUsers();
+ } catch (Exception e) {
+ // unable to access local cache... start up failure
+ logger.error(String.format("Unable to get existing user base. Cannot proceed until these users can be "
+ + "verified against the current authority provider: %s", e));
+ throw new AdministrationException(e);
+ }
+
+ try {
+ // all users for all roles
+ for (final Authority authority : Authority.values()) {
+ authorizedDns.addAll(authorityProvider.getUsers(authority));
+ }
+ } catch (AuthorityAccessException aae) {
+ // unable to access the authority provider... honor the cache
+ logger.warn("Unable to access authority provider due to " + aae);
+ return null;
+ }
+
+ final Set<NiFiUser> accountsToRevoke = new HashSet<>(existingUsers);
+
+ // persist the users
+ for (String dn : authorizedDns) {
+ NiFiUser user = null;
+ try {
+ // locate the user for this dn
+ user = userDao.findUserByDn(dn);
+ boolean newAccount = false;
+
+ // if the user does not exist, create a new account
+ if (user == null) {
+ logger.info(String.format("Creating user account: %s", dn));
+ newAccount = true;
+
+ // create the user
+ user = new NiFiUser();
+ user.setDn(dn);
+ user.setUserName(CertificateUtils.extractUsername(dn));
+ user.setJustification("User details specified by authority provider.");
+ } else {
+ logger.info(String.format("User account already created: %s. Updating authorities...", dn));
+ }
+
+ // verify the account
+ verifyAccount(authorityProvider, user);
+
+ // persist the account accordingly
+ if (newAccount) {
+ CreateUserAction createUser = new CreateUserAction(user);
+ createUser.execute(daoFactory, authorityProvider);
+ } else {
+ // this is not a new user and we have just verified their
+ // account, do not revoke...
+ accountsToRevoke.remove(user);
+
+ // persist the user
+ UpdateUserCacheAction updateUser = new UpdateUserCacheAction(user);
+ updateUser.execute(daoFactory, authorityProvider);
+
+ // persist the user's authorities
+ UpdateUserAuthoritiesCacheAction updateUserAuthorities = new UpdateUserAuthoritiesCacheAction(user);
+ updateUserAuthorities.execute(daoFactory, authorityProvider);
+ }
+ } catch (DataAccessException dae) {
+ if (user != null) {
+ logger.warn(String.format("Unable to access account details in local cache for user %s: %s", user, dae.getMessage()));
+ } else {
+ logger.warn(String.format("Unable to access account details in local cache: %s", dae.getMessage()));
+ }
+ } catch (UnknownIdentityException uie) {
+ if (user != null) {
+ logger.warn(String.format("Unable to find account details in authority provider for user %s: %s", user, uie.getMessage()));
+ } else {
+ logger.warn(String.format("Unable to find account details in authority provider: %s", uie.getMessage()));
+ }
+ } catch (AuthorityAccessException aae) {
+ logger.warn("Unable to access authority provider due to " + aae);
+
+ // unable to access authority provider for this user, honor the cache for now
+ accountsToRevoke.remove(user);
+ }
+ }
+
+ // remove all users that are no longer in the provider
+ for (final NiFiUser user : accountsToRevoke) {
+ // allow pending requests to remain...
+ if (AccountStatus.PENDING.equals(user.getStatus())) {
+ continue;
+ }
+
+ try {
+ logger.info(String.format("User not authorized with configured provider: %s. Disabling account...", user.getDn()));
+
+ // disable the account and reset its last verified timestamp since it was not found
+ // in the current configured authority provider
+ user.setStatus(AccountStatus.DISABLED);
+ user.setLastVerified(null);
+
+ // update the user record
+ UpdateUserCacheAction updateUser = new UpdateUserCacheAction(user);
+ updateUser.execute(daoFactory, authorityProvider);
+ } catch (final Exception e) {
+ // unable to revoke access for someone we know is not authorized... fail start up
+ logger.error(String.format("Unable to revoke access for user %s that is no longer authorized: %s", user, e));
+ throw new AdministrationException(e);
+ }
+ }
+
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UngroupUserAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UngroupUserAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UngroupUserAction.java
new file mode 100644
index 0000000..01eaf5f
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UngroupUserAction.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.admin.dao.UserDAO;
+import org.apache.nifi.admin.service.AccountNotFoundException;
+import org.apache.nifi.admin.service.AdministrationException;
+import org.apache.nifi.authorization.AuthorityProvider;
+import org.apache.nifi.authorization.exception.AuthorityAccessException;
+import org.apache.nifi.authorization.exception.UnknownIdentityException;
+import org.apache.nifi.user.NiFiUser;
+
+/**
+ *
+ */
+public class UngroupUserAction extends AbstractUserAction<Void> {
+
+ private final String userId;
+
+ public UngroupUserAction(String userId) {
+ this.userId = userId;
+ }
+
+ @Override
+ public Void execute(DAOFactory daoFactory, AuthorityProvider authorityProvider) {
+ final UserDAO userDao = daoFactory.getUserDAO();
+
+ // get the user in question
+ final NiFiUser user = userDao.findUserById(userId);
+
+ // ensure the user exists
+ if (user == null) {
+ throw new AccountNotFoundException(String.format("Unable to find account with ID %s.", userId));
+ }
+
+ // set the user group
+ user.setUserGroup(null);
+
+ // update the user locally
+ userDao.updateUser(user);
+
+ try {
+ // update the authority provider
+ authorityProvider.ungroupUser(user.getDn());
+ } catch (UnknownIdentityException uie) {
+ throw new AccountNotFoundException(String.format("Unable to ungroup user '%s': %s", user.getDn(), uie.getMessage()), uie);
+ } catch (AuthorityAccessException aae) {
+ throw new AdministrationException(String.format("Unable to ungroup user '%s': %s", user.getDn(), aae.getMessage()), aae);
+ }
+
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UngroupUserGroupAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UngroupUserGroupAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UngroupUserGroupAction.java
new file mode 100644
index 0000000..fa24fbe
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UngroupUserGroupAction.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.admin.dao.UserDAO;
+import org.apache.nifi.admin.service.AccountNotFoundException;
+import org.apache.nifi.admin.service.AdministrationException;
+import org.apache.nifi.authorization.AuthorityProvider;
+import org.apache.nifi.authorization.exception.AuthorityAccessException;
+import org.apache.nifi.authorization.exception.UnknownIdentityException;
+
+/**
+ *
+ */
+public class UngroupUserGroupAction extends AbstractUserAction<Void> {
+
+ private final String group;
+
+ public UngroupUserGroupAction(String group) {
+ this.group = group;
+ }
+
+ @Override
+ public Void execute(DAOFactory daoFactory, AuthorityProvider authorityProvider) {
+ final UserDAO userDao = daoFactory.getUserDAO();
+
+ // update the user locally
+ userDao.ungroup(group);
+
+ try {
+ // update the authority provider
+ authorityProvider.ungroup(group);
+ } catch (UnknownIdentityException uie) {
+ throw new AccountNotFoundException(String.format("Unable to ungroup '%s': %s", group, uie.getMessage()), uie);
+ } catch (AuthorityAccessException aae) {
+ throw new AdministrationException(String.format("Unable to ungroup '%s': %s", group, aae.getMessage()), aae);
+ }
+
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UpdateUserAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UpdateUserAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UpdateUserAction.java
new file mode 100644
index 0000000..cef21d7
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UpdateUserAction.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import java.util.Date;
+import java.util.Set;
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.admin.dao.DataAccessException;
+import org.apache.nifi.admin.dao.UserDAO;
+import org.apache.nifi.admin.service.AccountNotFoundException;
+import org.apache.nifi.admin.service.AdministrationException;
+import org.apache.nifi.authorization.Authority;
+import org.apache.nifi.authorization.AuthorityProvider;
+import org.apache.nifi.authorization.exception.AuthorityAccessException;
+import org.apache.nifi.authorization.exception.IdentityAlreadyExistsException;
+import org.apache.nifi.authorization.exception.UnknownIdentityException;
+import org.apache.nifi.user.AccountStatus;
+import org.apache.nifi.user.NiFiUser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Sets user authorities.
+ */
+public class UpdateUserAction extends AbstractUserAction<NiFiUser> {
+
+ private static final Logger logger = LoggerFactory.getLogger(UpdateUserAction.class);
+
+ private final String id;
+ private final Set<Authority> authorities;
+
+ public UpdateUserAction(String id, Set<Authority> authorities) {
+ this.id = id;
+ this.authorities = authorities;
+ }
+
+ @Override
+ public NiFiUser execute(DAOFactory daoFactory, AuthorityProvider authorityProvider) throws DataAccessException, AdministrationException {
+ UserDAO userDao = daoFactory.getUserDAO();
+
+ // get the user
+ NiFiUser user = userDao.findUserById(id);
+
+ // ensure the user exists
+ if (user == null) {
+ throw new AccountNotFoundException(String.format("Unable to find account with ID %s.", id));
+ }
+
+ // determine whether this users exists
+ boolean doesDnExist = false;
+ try {
+ doesDnExist = authorityProvider.doesDnExist(user.getDn());
+ } catch (AuthorityAccessException aae) {
+ throw new AdministrationException(String.format("Unable to access authority details: %s", aae.getMessage()), aae);
+ }
+
+ // if the user already doesn't exist, add them
+ if (!doesDnExist) {
+ try {
+ // add the account account and group if necessary
+ authorityProvider.addUser(user.getDn(), user.getUserGroup());
+ } catch (final IdentityAlreadyExistsException iaee) {
+ logger.warn(String.format("User '%s' already exists in the authority provider. Continuing with user update.", user.getDn()));
+ } catch (AuthorityAccessException aae) {
+ throw new AdministrationException(String.format("Unable to access authorities for '%s': %s", user.getDn(), aae.getMessage()), aae);
+ }
+ }
+
+ try {
+ // update the authority provider as approprivate
+ authorityProvider.setAuthorities(user.getDn(), authorities);
+ } catch (UnknownIdentityException uie) {
+ throw new AccountNotFoundException(String.format("Unable to modify authorities for '%s': %s.", user.getDn(), uie.getMessage()), uie);
+ } catch (AuthorityAccessException aae) {
+ throw new AdministrationException(String.format("Unable to access authorities for '%s': %s.", user.getDn(), aae.getMessage()), aae);
+ }
+
+ try {
+ // get the user group
+ user.setUserGroup(authorityProvider.getGroupForUser(user.getDn()));
+ } catch (UnknownIdentityException uie) {
+ throw new AccountNotFoundException(String.format("Unable to determine the group for '%s': %s.", user.getDn(), uie.getMessage()), uie);
+ } catch (AuthorityAccessException aae) {
+ throw new AdministrationException(String.format("Unable to access the group for '%s': %s.", user.getDn(), aae.getMessage()), aae);
+ }
+
+ // since all the authorities were updated accordingly, set the authorities
+ user.getAuthorities().clear();
+ user.getAuthorities().addAll(authorities);
+
+ // update the users status in case they were previously pending or disabled
+ user.setStatus(AccountStatus.ACTIVE);
+
+ // update the users last verified time - this timestamp shouldn't be recorded
+ // until the both the user's authorities and group have been synced
+ Date now = new Date();
+ user.setLastVerified(now);
+
+ // persist the user's updates
+ UpdateUserCacheAction updateUser = new UpdateUserCacheAction(user);
+ updateUser.execute(daoFactory, authorityProvider);
+
+ // persist the user's authorities
+ UpdateUserAuthoritiesCacheAction updateUserAuthorities = new UpdateUserAuthoritiesCacheAction(user);
+ updateUserAuthorities.execute(daoFactory, authorityProvider);
+
+ // return the user
+ return user;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UpdateUserAuthoritiesCacheAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UpdateUserAuthoritiesCacheAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UpdateUserAuthoritiesCacheAction.java
new file mode 100644
index 0000000..89661b2
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UpdateUserAuthoritiesCacheAction.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import java.util.Set;
+import org.apache.nifi.admin.dao.AuthorityDAO;
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.admin.dao.DataAccessException;
+import org.apache.nifi.admin.dao.UserDAO;
+import org.apache.nifi.admin.service.AccountNotFoundException;
+import org.apache.nifi.authorization.Authority;
+import org.apache.nifi.authorization.AuthorityProvider;
+import org.apache.nifi.user.NiFiUser;
+import org.apache.commons.collections4.CollectionUtils;
+
+/**
+ * Updates a NiFiUser's authorities. Prior to invoking this action, the user's
+ * authorities should be set according to the business logic of the service in
+ * question. This should not be invoked directly when attempting to set user
+ * authorities as the authorityProvider is not called from this action.
+ */
+public class UpdateUserAuthoritiesCacheAction extends AbstractUserAction<Void> {
+
+ private final NiFiUser user;
+
+ public UpdateUserAuthoritiesCacheAction(NiFiUser user) {
+ this.user = user;
+ }
+
+ @Override
+ public Void execute(DAOFactory daoFactory, AuthorityProvider authorityProvider) throws DataAccessException {
+ UserDAO userDao = daoFactory.getUserDAO();
+ AuthorityDAO authorityDao = daoFactory.getAuthorityDAO();
+
+ // get the user
+ NiFiUser currentUser = userDao.findUserById(user.getId());
+
+ // ensure the user exists
+ if (currentUser == null) {
+ throw new AccountNotFoundException(String.format("Unable to find account with ID %s.", user.getId()));
+ }
+
+ // determine what authorities need to be added/removed
+ Set<Authority> authorities = user.getAuthorities();
+ Set<Authority> authoritiesToAdd = determineAuthoritiesToAdd(currentUser, authorities);
+ Set<Authority> authoritiesToRemove = determineAuthoritiesToRemove(currentUser, authorities);
+
+ // update the user authorities locally
+ if (CollectionUtils.isNotEmpty(authoritiesToAdd)) {
+ authorityDao.createAuthorities(authoritiesToAdd, user.getId());
+ }
+ if (CollectionUtils.isNotEmpty(authoritiesToRemove)) {
+ authorityDao.deleteAuthorities(authoritiesToRemove, user.getId());
+ }
+
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UpdateUserCacheAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UpdateUserCacheAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UpdateUserCacheAction.java
new file mode 100644
index 0000000..288e297
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UpdateUserCacheAction.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.admin.dao.DataAccessException;
+import org.apache.nifi.admin.dao.UserDAO;
+import org.apache.nifi.authorization.AuthorityProvider;
+import org.apache.nifi.user.NiFiUser;
+
+/**
+ * Updates a NiFiUser. This will not update the user authorities, they must be
+ * updated with the UpdateUserAuthoritiesAction.
+ */
+public class UpdateUserCacheAction extends AbstractUserAction<Void> {
+
+ private final NiFiUser user;
+
+ public UpdateUserCacheAction(NiFiUser user) {
+ this.user = user;
+ }
+
+ @Override
+ public Void execute(DAOFactory daoFactory, AuthorityProvider authorityProvider) throws DataAccessException {
+ UserDAO userDao = daoFactory.getUserDAO();
+
+ // update the user
+ userDao.updateUser(user);
+
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UpdateUserGroupAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UpdateUserGroupAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UpdateUserGroupAction.java
new file mode 100644
index 0000000..56b214c
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UpdateUserGroupAction.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import java.util.Date;
+import java.util.HashSet;
+import java.util.Set;
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.admin.dao.DataAccessException;
+import org.apache.nifi.admin.dao.UserDAO;
+import org.apache.nifi.admin.service.AccountNotFoundException;
+import org.apache.nifi.admin.service.AdministrationException;
+import org.apache.nifi.authorization.Authority;
+import org.apache.nifi.authorization.AuthorityProvider;
+import org.apache.nifi.authorization.exception.AuthorityAccessException;
+import org.apache.nifi.authorization.exception.UnknownIdentityException;
+import org.apache.nifi.user.AccountStatus;
+import org.apache.nifi.user.NiFiUser;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Updates all NiFiUser authorities in a specified group.
+ */
+public class UpdateUserGroupAction extends AbstractUserAction<Void> {
+
+ private static final Logger logger = LoggerFactory.getLogger(UpdateUserGroupAction.class);
+
+ private final String group;
+ private final Set<String> userIds;
+ private final Set<Authority> authorities;
+
+ public UpdateUserGroupAction(String group, Set<String> userIds, Set<Authority> authorities) {
+ this.group = group;
+ this.userIds = userIds;
+ this.authorities = authorities;
+ }
+
+ @Override
+ public Void execute(DAOFactory daoFactory, AuthorityProvider authorityProvider) throws DataAccessException {
+ if (userIds == null && authorities == null) {
+ throw new IllegalArgumentException("Must specify user Ids or authorities.");
+ }
+
+ UserDAO userDao = daoFactory.getUserDAO();
+
+ // record the new users being added to this group
+ final Set<NiFiUser> newUsers = new HashSet<>();
+ final Set<String> newUserDns = new HashSet<>();
+
+ // if the user ids have been specified we need to create/update a group using the specified group name
+ if (userIds != null) {
+ if (userIds.isEmpty()) {
+ throw new IllegalArgumentException("When creating a group, at least one user id must be specified.");
+ }
+
+ // going to create a group using the specified user ids
+ for (final String userId : userIds) {
+ // get the user in question
+ final NiFiUser user = userDao.findUserById(userId);
+
+ // ensure the user exists
+ if (user == null) {
+ throw new AccountNotFoundException(String.format("Unable to find account with ID %s.", userId));
+ }
+
+ try {
+ // if the user is unknown to the authority provider we cannot continue
+ if (!authorityProvider.doesDnExist(user.getDn()) || AccountStatus.DISABLED.equals(user.getStatus())) {
+ throw new IllegalStateException(String.format("Unable to group these users because access for '%s' is not %s.", user.getDn(), AccountStatus.ACTIVE.toString()));
+ }
+
+ // record the user being added to this group
+ newUsers.add(user);
+ newUserDns.add(user.getDn());
+ } catch (final AuthorityAccessException aae) {
+ throw new AdministrationException(String.format("Unable to access authority details: %s", aae.getMessage()), aae);
+ }
+ }
+
+ try {
+ // update the authority provider
+ authorityProvider.setUsersGroup(newUserDns, group);
+ } catch (UnknownIdentityException uie) {
+ throw new AccountNotFoundException(String.format("Unable to set user group '%s': %s", StringUtils.join(newUserDns, ", "), uie.getMessage()), uie);
+ } catch (AuthorityAccessException aae) {
+ throw new AdministrationException(String.format("Unable to set user group '%s': %s", StringUtils.join(newUserDns, ", "), aae.getMessage()), aae);
+ }
+ }
+
+ // get all the users that need to be updated
+ final Set<NiFiUser> users = new HashSet<>(userDao.findUsersForGroup(group));
+ users.addAll(newUsers);
+
+ // ensure the user exists
+ if (users.isEmpty()) {
+ throw new AccountNotFoundException(String.format("Unable to find user accounts with group id %s.", group));
+ }
+
+ // update each user in this group
+ for (final NiFiUser user : users) {
+ // if there are new authorities set them, otherwise refresh them according to the provider
+ if (authorities != null) {
+ try {
+ // update the authority provider as approprivate
+ authorityProvider.setAuthorities(user.getDn(), authorities);
+
+ // since all the authorities were updated accordingly, set the authorities
+ user.getAuthorities().clear();
+ user.getAuthorities().addAll(authorities);
+ } catch (UnknownIdentityException uie) {
+ throw new AccountNotFoundException(String.format("Unable to modify authorities for '%s': %s.", user.getDn(), uie.getMessage()), uie);
+ } catch (AuthorityAccessException aae) {
+ throw new AdministrationException(String.format("Unable to access authorities for '%s': %s.", user.getDn(), aae.getMessage()), aae);
+ }
+ } else {
+ try {
+ // refresh the authorities according to the provider
+ user.getAuthorities().clear();
+ user.getAuthorities().addAll(authorityProvider.getAuthorities(user.getDn()));
+ } catch (UnknownIdentityException uie) {
+ throw new AccountNotFoundException(String.format("Unable to determine the authorities for '%s': %s.", user.getDn(), uie.getMessage()), uie);
+ } catch (AuthorityAccessException aae) {
+ throw new AdministrationException(String.format("Unable to access authorities for '%s': %s.", user.getDn(), aae.getMessage()), aae);
+ }
+ }
+
+ try {
+ // get the user group
+ user.setUserGroup(authorityProvider.getGroupForUser(user.getDn()));
+ } catch (UnknownIdentityException uie) {
+ throw new AccountNotFoundException(String.format("Unable to determine the group for '%s': %s.", user.getDn(), uie.getMessage()), uie);
+ } catch (AuthorityAccessException aae) {
+ throw new AdministrationException(String.format("Unable to access the group for '%s': %s.", user.getDn(), aae.getMessage()), aae);
+ }
+
+ // update the users status in case they were previously pending or disabled
+ user.setStatus(AccountStatus.ACTIVE);
+
+ // update the users last verified time - this timestamp shouldn't be recorded
+ // until the both the user's authorities and group have been synced
+ Date now = new Date();
+ user.setLastVerified(now);
+
+ // persist the user's updates
+ UpdateUserCacheAction updateUser = new UpdateUserCacheAction(user);
+ updateUser.execute(daoFactory, authorityProvider);
+
+ // persist the user's authorities
+ UpdateUserAuthoritiesCacheAction updateUserAuthorities = new UpdateUserAuthoritiesCacheAction(user);
+ updateUserAuthorities.execute(daoFactory, authorityProvider);
+ }
+
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/impl/StandardAuditService.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/impl/StandardAuditService.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/impl/StandardAuditService.java
new file mode 100644
index 0000000..e588841
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/impl/StandardAuditService.java
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.impl;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.nifi.action.Action;
+import org.apache.nifi.admin.dao.DataAccessException;
+import org.apache.nifi.admin.service.AdministrationException;
+import org.apache.nifi.admin.service.AuditService;
+import org.apache.nifi.admin.service.action.AddActionsAction;
+import org.apache.nifi.admin.service.action.GetActionAction;
+import org.apache.nifi.admin.service.action.GetActionsAction;
+import org.apache.nifi.admin.service.action.GetPreviousValues;
+import org.apache.nifi.admin.service.action.PurgeActionsAction;
+import org.apache.nifi.admin.service.transaction.Transaction;
+import org.apache.nifi.admin.service.transaction.TransactionBuilder;
+import org.apache.nifi.admin.service.transaction.TransactionException;
+import org.apache.nifi.history.History;
+import org.apache.nifi.history.HistoryQuery;
+import org.apache.nifi.history.PreviousValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ */
+public class StandardAuditService implements AuditService {
+
+ private static final Logger logger = LoggerFactory.getLogger(StandardAuditService.class);
+
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+ private final ReentrantReadWriteLock.ReadLock readLock = lock.readLock();
+ private final ReentrantReadWriteLock.WriteLock writeLock = lock.writeLock();
+
+ private TransactionBuilder transactionBuilder;
+
+ @Override
+ public void addActions(Collection<Action> actions) {
+ Transaction transaction = null;
+
+ writeLock.lock();
+ try {
+ // start the transaction
+ transaction = transactionBuilder.start();
+
+ // seed the accounts
+ AddActionsAction addActions = new AddActionsAction(actions);
+ transaction.execute(addActions);
+
+ // commit the transaction
+ transaction.commit();
+ } catch (TransactionException | DataAccessException te) {
+ rollback(transaction);
+ throw new AdministrationException(te);
+ } catch (Throwable t) {
+ rollback(transaction);
+ throw t;
+ } finally {
+ closeQuietly(transaction);
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public Map<String, List<PreviousValue>> getPreviousValues(String componentId) {
+ Transaction transaction = null;
+ Map<String, List<PreviousValue>> previousValues = null;
+
+ readLock.lock();
+ try {
+ // start the transaction
+ transaction = transactionBuilder.start();
+
+ // seed the accounts
+ GetPreviousValues getActions = new GetPreviousValues(componentId);
+ previousValues = transaction.execute(getActions);
+
+ // commit the transaction
+ transaction.commit();
+ } catch (TransactionException | DataAccessException te) {
+ rollback(transaction);
+ throw new AdministrationException(te);
+ } catch (Throwable t) {
+ rollback(transaction);
+ throw t;
+ } finally {
+ closeQuietly(transaction);
+ readLock.unlock();
+ }
+
+ return previousValues;
+ }
+
+ @Override
+ public History getActions(HistoryQuery query) {
+ Transaction transaction = null;
+ History history = null;
+
+ readLock.lock();
+ try {
+ // start the transaction
+ transaction = transactionBuilder.start();
+
+ // seed the accounts
+ GetActionsAction getActions = new GetActionsAction(query);
+ history = transaction.execute(getActions);
+
+ // commit the transaction
+ transaction.commit();
+ } catch (TransactionException | DataAccessException te) {
+ rollback(transaction);
+ throw new AdministrationException(te);
+ } catch (Throwable t) {
+ rollback(transaction);
+ throw t;
+ } finally {
+ closeQuietly(transaction);
+ readLock.unlock();
+ }
+
+ return history;
+ }
+
+ @Override
+ public Action getAction(Integer actionId) {
+ Transaction transaction = null;
+ Action action = null;
+
+ readLock.lock();
+ try {
+ // start the transaction
+ transaction = transactionBuilder.start();
+
+ // seed the accounts
+ GetActionAction getAction = new GetActionAction(actionId);
+ action = transaction.execute(getAction);
+
+ // commit the transaction
+ transaction.commit();
+ } catch (TransactionException | DataAccessException te) {
+ rollback(transaction);
+ throw new AdministrationException(te);
+ } catch (Throwable t) {
+ rollback(transaction);
+ throw t;
+ } finally {
+ closeQuietly(transaction);
+ readLock.unlock();
+ }
+
+ return action;
+ }
+
+ @Override
+ public void purgeActions(Date end, Action purgeAction) {
+ Transaction transaction = null;
+
+ writeLock.lock();
+ try {
+ // start the transaction
+ transaction = transactionBuilder.start();
+
+ // purge the action database
+ PurgeActionsAction purgeActions = new PurgeActionsAction(end, purgeAction);
+ transaction.execute(purgeActions);
+
+ // commit the transaction
+ transaction.commit();
+ } catch (TransactionException | DataAccessException te) {
+ rollback(transaction);
+ throw new AdministrationException(te);
+ } catch (Throwable t) {
+ rollback(transaction);
+ throw t;
+ } finally {
+ closeQuietly(transaction);
+ writeLock.unlock();
+ }
+ }
+
+ private void rollback(Transaction transaction) {
+ if (transaction != null) {
+ transaction.rollback();
+ }
+ }
+
+ private void closeQuietly(final Transaction transaction) {
+ if (transaction != null) {
+ try {
+ transaction.close();
+ } catch (final IOException ioe) {
+ }
+ }
+ }
+
+ public void setTransactionBuilder(TransactionBuilder transactionBuilder) {
+ this.transactionBuilder = transactionBuilder;
+ }
+
+}
[07/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/NifiSinkSessionChannel.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/NifiSinkSessionChannel.java b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/NifiSinkSessionChannel.java
new file mode 100644
index 0000000..5621b6d
--- /dev/null
+++ b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/NifiSinkSessionChannel.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.flume;
+
+import org.apache.flume.Context;
+import org.apache.flume.channel.BasicChannelSemantics;
+import org.apache.flume.channel.BasicTransactionSemantics;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+
+public class NifiSinkSessionChannel extends BasicChannelSemantics {
+
+ private ProcessSession session;
+ private final Relationship success;
+ private final Relationship failure;
+
+ public NifiSinkSessionChannel(Relationship success, Relationship failure) {
+ this.success = success;
+ this.failure = failure;
+ }
+
+ public void setSession(ProcessSession session) {
+ this.session = session;
+ }
+
+ @Override
+ protected BasicTransactionSemantics createTransaction() {
+ return new NifiSinkTransaction(session, success, failure);
+ }
+
+ @Override
+ public void configure(Context context) {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/NifiSinkTransaction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/NifiSinkTransaction.java b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/NifiSinkTransaction.java
new file mode 100644
index 0000000..837652f
--- /dev/null
+++ b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/NifiSinkTransaction.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.flume;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.flume.Event;
+import org.apache.flume.channel.BasicTransactionSemantics;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processors.flume.util.FlowFileEvent;
+
+
+class NifiSinkTransaction extends BasicTransactionSemantics {
+ private final ProcessSession session;
+ private final Relationship success;
+ private final Relationship failure;
+ private final List<FlowFile> flowFiles;
+
+ public NifiSinkTransaction(ProcessSession session, Relationship success, Relationship failure) {
+ this.session = session;
+ this.success = success;
+ this.failure = failure;
+ this.flowFiles = new ArrayList<>();
+ }
+
+ @Override
+ protected void doPut(Event event) throws InterruptedException {
+ AbstractFlumeProcessor.transferEvent(event, session, success);
+ }
+
+ @Override
+ protected Event doTake() throws InterruptedException {
+ FlowFile flowFile = session.get();
+ if (flowFile == null) {
+ return null;
+ }
+ flowFiles.add(flowFile);
+
+ return new FlowFileEvent(flowFile, session);
+ }
+
+ @Override
+ protected void doCommit() throws InterruptedException {
+ session.transfer(flowFiles, success);
+ session.commit();
+ }
+
+ @Override
+ protected void doRollback() throws InterruptedException {
+ session.transfer(flowFiles, failure);
+ session.commit();
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/NifiTransaction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/NifiTransaction.java b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/NifiTransaction.java
new file mode 100644
index 0000000..8de50ec
--- /dev/null
+++ b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/NifiTransaction.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.flume;
+
+import org.apache.flume.Event;
+import org.apache.flume.channel.BasicTransactionSemantics;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+
+class NifiTransaction extends BasicTransactionSemantics {
+
+ private final ProcessSession session;
+ private final Relationship relationship;
+
+ public NifiTransaction(ProcessSession session, Relationship relationship) {
+ this.session = session;
+ this.relationship = relationship;
+ }
+
+ @Override
+ protected void doPut(Event event) throws InterruptedException {
+ AbstractFlumeProcessor.transferEvent(event, session, relationship);
+ }
+
+ @Override
+ protected Event doTake() throws InterruptedException {
+ throw new UnsupportedOperationException("Only put supported");
+ }
+
+ @Override
+ protected void doCommit() throws InterruptedException {
+ session.commit();
+ }
+
+ @Override
+ protected void doRollback() throws InterruptedException {
+ session.rollback();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/util/FlowFileEvent.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/util/FlowFileEvent.java b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/util/FlowFileEvent.java
new file mode 100644
index 0000000..fdff203
--- /dev/null
+++ b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/util/FlowFileEvent.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.flume.util;
+
+import com.google.common.collect.Maps;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
+import java.util.Map;
+import org.apache.flume.Event;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.io.InputStreamCallback;
+import static org.apache.nifi.processors.flume.util.FlowFileEventConstants.ENTRY_DATE_HEADER;
+import static org.apache.nifi.processors.flume.util.FlowFileEventConstants.ID_HEADER;
+import static org.apache.nifi.processors.flume.util.FlowFileEventConstants.LAST_QUEUE_DATE_HEADER;
+import static org.apache.nifi.processors.flume.util.FlowFileEventConstants.LINEAGE_IDENTIFIERS_HEADER;
+import static org.apache.nifi.processors.flume.util.FlowFileEventConstants.LINEAGE_START_DATE_HEADER;
+import static org.apache.nifi.processors.flume.util.FlowFileEventConstants.SIZE_HEADER;
+
+import org.apache.nifi.stream.io.BufferedInputStream;
+import org.apache.nifi.stream.io.StreamUtils;
+
+public class FlowFileEvent implements Event {
+
+ private final FlowFile flowFile;
+ private final ProcessSession session;
+
+ private final Map<String, String> headers;
+ private boolean headersLoaded;
+
+ private final Object bodyLock;
+ private byte[] body;
+ private boolean bodyLoaded;
+
+ public FlowFileEvent(FlowFile flowFile, ProcessSession session) {
+ this.flowFile = flowFile;
+ this.session = session;
+
+ headers = Maps.newHashMap();
+ bodyLock = new Object();
+ bodyLoaded = false;
+ }
+
+ @Override
+ public Map<String, String> getHeaders() {
+ if (!headersLoaded) {
+ synchronized (headers) {
+ if (headersLoaded) {
+ return headers;
+ }
+
+ headers.putAll(flowFile.getAttributes());
+ headers.put(ENTRY_DATE_HEADER, Long.toString(flowFile.getEntryDate()));
+ headers.put(ID_HEADER, Long.toString(flowFile.getId()));
+ headers.put(LAST_QUEUE_DATE_HEADER, Long.toString(flowFile.getLastQueueDate()));
+ int i = 0;
+ for (String lineageIdentifier : flowFile.getLineageIdentifiers()) {
+ headers.put(LINEAGE_IDENTIFIERS_HEADER + "." + i, lineageIdentifier);
+ i++;
+ }
+ headers.put(LINEAGE_START_DATE_HEADER, Long.toString(flowFile.getLineageStartDate()));
+ headers.put(SIZE_HEADER, Long.toString(flowFile.getSize()));
+
+ headersLoaded = true;
+ }
+ }
+ return headers;
+ }
+
+ @Override
+ public void setHeaders(Map<String, String> headers) {
+ synchronized (this.headers) {
+ this.headers.clear();
+ this.headers.putAll(headers);
+ headersLoaded = true;
+ }
+ }
+
+ @Override
+ public byte[] getBody() {
+ if (bodyLoaded) {
+ return body;
+ }
+
+ synchronized (bodyLock ) {
+ if (!bodyLoaded) {
+ if (flowFile.getSize() > Integer.MAX_VALUE) {
+ throw new RuntimeException("Can't get body of Event because the backing FlowFile is too large (" + flowFile.getSize() + " bytes)");
+ }
+
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream((int) flowFile.getSize());
+ session.read(flowFile, new InputStreamCallback() {
+
+ @Override
+ public void process(InputStream in) throws IOException {
+ try (BufferedInputStream input = new BufferedInputStream(in)) {
+ StreamUtils.copy(input, baos);
+ }
+ baos.close();
+ }
+ });
+
+ body = baos.toByteArray();
+ bodyLoaded = true;
+ }
+ }
+
+ return body;
+ }
+
+ @Override
+ public void setBody(byte[] body) {
+ synchronized (bodyLock) {
+ this.body = Arrays.copyOf(body, body.length);
+ bodyLoaded = true;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/util/FlowFileEventConstants.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/util/FlowFileEventConstants.java b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/util/FlowFileEventConstants.java
new file mode 100644
index 0000000..c9650c1
--- /dev/null
+++ b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/util/FlowFileEventConstants.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.flume.util;
+
+
+public class FlowFileEventConstants {
+
+ // FlowFile#getEntryDate();
+ public static final String ENTRY_DATE_HEADER = "nifi.entry.date";
+
+ // FlowFile#getId();
+ public static final String ID_HEADER = "nifi.id";
+
+ // FlowFile#getLastQueueDate();
+ public static final String LAST_QUEUE_DATE_HEADER = "nifi.last.queue.date";
+
+ // FlowFile#getLineageIdentifiers();
+ public static final String LINEAGE_IDENTIFIERS_HEADER = "nifi.lineage.identifiers";
+
+ // FlowFile#getLineageStartDate();
+ public static final String LINEAGE_START_DATE_HEADER = "nifi.lineage.start.date";
+
+ // FlowFile#getSize();
+ public static final String SIZE_HEADER = "nifi.size";
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
new file mode 100644
index 0000000..f5b57e1
--- /dev/null
+++ b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+org.apache.nifi.processors.flume.ExecuteFlumeSource
+org.apache.nifi.processors.flume.ExecuteFlumeSink
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/resources/docs/org.apache.nifi.processors.flume.ExecuteFlumeSink/additionalDetails.html
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/resources/docs/org.apache.nifi.processors.flume.ExecuteFlumeSink/additionalDetails.html b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/resources/docs/org.apache.nifi.processors.flume.ExecuteFlumeSink/additionalDetails.html
new file mode 100644
index 0000000..be8b99f
--- /dev/null
+++ b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/resources/docs/org.apache.nifi.processors.flume.ExecuteFlumeSink/additionalDetails.html
@@ -0,0 +1,159 @@
+<!DOCTYPE html>
+<html lang="en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<head>
+ <meta charset="utf-8" />
+ <title>ExecuteFlumeSink</title>
+ <link rel="stylesheet" href="../../css/component-usage.css" type="text/css" />
+</head>
+
+<body>
+<h2>Data Model</h2>
+<p>
+ This processor executes an Apache Flume sink. FlowFiles are wrapped in Flume's
+ Event interface. The content of the FlowFile becomes the body of the Event and
+ the attributes of the FlowFile become Event headers. The following special
+ headers are also set:
+</p>
+<table id="headers">
+ <tr>
+ <th>Flume Event Header</th>
+ <th>FlowFile Attribute</th>
+ </tr>
+ <tr>
+ <td>nifi.entry.date</td>
+ <td>FlowFile#getEntryDate()</td>
+ </tr>
+ <tr>
+ <td>nifi.id</td>
+ <td>FlowFile#getId()</td>
+ </tr>
+ <tr>
+ <td>nifi.last.queue.date</td>
+ <td>FlowFile#getLastQueueDate()</td>
+ </tr>
+ <tr>
+ <td>nifi.lineage.identifiers.${i}</td>
+ <td>FlowFile#getLineageIdentifiers()[i]</td>
+ </tr>
+ <tr>
+ <td>nifi.lineage.start.date</td>
+ <td>FlowFile#getLineageStartDate()</td>
+ </tr>
+ <tr>
+ <td>nifi.size</td>
+ <td>FlowFile#getSize()</td>
+ </tr>
+</table>
+<h2>Warning</h2>
+<p>
+ In NiFi, the contents of a FlowFile are accessed via a stream, but in Flume it is
+ stored in a byte array. This means the full content will be loaded into memory when
+ a FlowFile is processed by the ExecuteFlumeSink processor. You should consider the
+ typical size of the FlowFiles you'll process and the batch size, if any, your sink
+ is configured with when setting NiFi's heap size.
+</p>
+<h2>Configuration Details</h2>
+<p>
+ This processor is designed to execute arbitrary Flume sinks. Most of the details
+ of configuring the sink is deferred to Flume's built-in configuration system.
+ For details on the available settings for each sink type, refer to the Flume
+ <a href="http://flume.apache.org/FlumeUserGuide.html#flume-sinks">User Guide</a>.
+ Configuring the Flume sink is a four step process:
+</p>
+<ol>
+ <li>Set the Sink Type property to a valid Flume sink type.</li>
+ <li>
+ Set the Agent Name property to the name of the agent in your
+ Flume configuration. This is the prefix of the properties in the Flume
+ configuration file. Example: <code>tier1</code>
+ </li>
+ <li>
+ Set the Sink Name property to the name of the sink in your Flume
+ configuration. If Agent Name is <code>tier1</code>, then the Sink Name
+ is the value of the <code>tier1.sinks</code> property. Example: <code>sink-1</code>
+ </li>
+ <li>
+ Copy and paste the configuration for the sink from your Flume configuration
+ file into the Flume Configuration property. Assuming you're using
+ the same Agent Name and Sink Name as in the examples above, this will be all
+ of the properties that start with <code>tier1.sinks.sink-1</code>.
+ Do not copy the <code>tier1.sinks.sink-1.type</code> or
+ <code>tier1.sinks.sink-1.channel</code> properties.
+ </li>
+</ol>
+<h2>Usage Example</h2>
+<p>
+ Assuming you had the following existing Flume configuration file:
+</p>
+ <pre>
+a1.sources = r1
+a1.sinks = k1
+a1.channels = c1
+
+a1.sources.r1.type = netcat
+a1.sources.r1.bind = localhost
+a1.sources.r1.port = 44444
+
+a1.sinks.k1.type = hdfs
+a1.sinks.k1.channel = c1
+a1.sinks.k1.hdfs.path = /flume/events/%y-%m-%d/%H%M/%S
+a1.sinks.k1.hdfs.filePrefix = events-
+a1.sinks.k1.hdfs.round = true
+a1.sinks.k1.hdfs.roundValue = 10
+a1.sinks.k1.hdfs.roundUnit = minute
+
+a1.channels.c1.type = memory
+a1.channels.c1.capacity = 1000
+a1.channels.c1.transactionCapacity = 100
+
+a1.sources.r1.channels = c1
+a1.sinks.k1.channel = c1</pre>
+<p>
+ Then you'd configure the ExecuteFlumeSink as follows:
+</p>
+<table id="example">
+ <tr>
+ <th>Property</th>
+ <th>Value</th>
+ </tr>
+ <tr>
+ <td>Sink Type</td>
+ <td>hdfs</td>
+ </tr>
+ <tr>
+ <td>Agent Name</td>
+ <td>a1</td>
+ </tr>
+ <tr>
+ <td>Sink Name</td>
+ <td>k1</td>
+ </tr>
+ <tr>
+ <td>Flume Configuration</td>
+ <td>
+ <code>
+ a1.sinks.k1.hdfs.path = /flume/events/%y-%m-%d/%H%M/%S<br>
+ a1.sinks.k1.hdfs.filePrefix = events-<br>
+ a1.sinks.k1.hdfs.round = true<br>
+ a1.sinks.k1.hdfs.roundValue = 10<br>
+ a1.sinks.k1.hdfs.roundUnit = minute
+ </code>
+ </td>
+ </tr>
+</table>
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/resources/docs/org.apache.nifi.processors.flume.ExecuteFlumeSource/additionalDetails.html
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/resources/docs/org.apache.nifi.processors.flume.ExecuteFlumeSource/additionalDetails.html b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/resources/docs/org.apache.nifi.processors.flume.ExecuteFlumeSource/additionalDetails.html
new file mode 100644
index 0000000..3d1062b
--- /dev/null
+++ b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/resources/docs/org.apache.nifi.processors.flume.ExecuteFlumeSource/additionalDetails.html
@@ -0,0 +1,114 @@
+<!DOCTYPE html>
+<html lang="en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<head>
+ <meta charset="utf-8" />
+ <title>ExecuteFlumeSource</title>
+ <link rel="stylesheet" href="../../css/component-usage.css" type="text/css" />
+</head>
+
+<body>
+<h2>Data Model</h2>
+<p>
+ This processor executes an Apache Flume source. Each Flume Event is turned into
+ a FlowFile. The content of the FlowFile is set to the body of the Event and
+ the Event headers become FlowFile attributes.
+</p>
+<h2>Configuration Details</h2>
+<p>
+ This processor is designed to execute arbitrary Flume sources. Most of the details
+ of configuring the source is deferred to Flume's built-in configuration system.
+ For details on the available settings for each source type, refer to the Flume
+ <a href="http://flume.apache.org/FlumeUserGuide.html#flume-sources">User Guide</a>.
+ Configuring the Flume source is a four step process:
+</p>
+<ol>
+ <li>Set the Source Type property to a valid Flume source type.</li>
+ <li>
+ Set the Agent Name property to the name of the agent in your
+ Flume configuration. This is the prefix of the properties in the Flume
+ configuration file. Example: <code>tier1</code>
+ </li>
+ <li>
+ Set the Source Name property to the name of the source in your Flume
+ configuration. If Agent Name is <code>tier1</code>, then the Source Name
+ is the value of the <code>tier1.sources</code> property. Example: <code>src-1</code>
+ </li>
+ <li>
+ Copy and paste the configuration for the source from your Flume configuration
+ file into the Flume Configuration property. Assuming you're using
+ the same Agent Name and Source Name as in the examples above, this will be all
+ of the properties that start with <code>tier1.sources.src-1</code>.
+ Do not copy the <code>tier1.sources.src-1.type</code> or
+ <code>tier1.sources.src-1.channel</code> properties.
+ </li>
+</ol>
+<h2>Usage Example</h2>
+<p>
+ Assuming you had the following existing Flume configuration file:
+</p>
+ <pre>
+a1.sources = r1
+a1.sinks = k1
+a1.channels = c1
+
+a1.sources.r1.type = multiport_syslogtcp
+a1.sources.r1.channels = c1
+a1.sources.r1.host = 0.0.0.0
+a1.sources.r1.ports = 10001 10002 10003
+a1.sources.r1.portHeader = port
+
+a1.sinks.k1.type = logger
+
+a1.channels.c1.type = memory
+a1.channels.c1.capacity = 1000
+a1.channels.c1.transactionCapacity = 100
+
+a1.sources.r1.channels = c1
+a1.sinks.k1.channel = c1</pre>
+<p>
+ Then you'd configure the ExecuteFlumeSource as follows:
+</p>
+<table id="example">
+ <tr>
+ <th>Property</th>
+ <th>Value</th>
+ </tr>
+ <tr>
+ <td>Source Type</td>
+ <td>multiport_syslogtcp</td>
+ </tr>
+ <tr>
+ <td>Agent Name</td>
+ <td>a1</td>
+ </tr>
+ <tr>
+ <td>Source Name</td>
+ <td>r1</td>
+ </tr>
+ <tr>
+ <td>Flume Configuration</td>
+ <td>
+ <code>
+ a1.sources.r1.host = 0.0.0.0<br>
+ a1.sources.r1.ports = 10001 10002 10003<br>
+ a1.sources.r1.portHeader = port
+ </code>
+ </td>
+ </tr>
+</table>
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/test/java/org/apache/nifi/processors/flume/ExecuteFlumeSinkTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/test/java/org/apache/nifi/processors/flume/ExecuteFlumeSinkTest.java b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/test/java/org/apache/nifi/processors/flume/ExecuteFlumeSinkTest.java
new file mode 100644
index 0000000..4d3bf4e
--- /dev/null
+++ b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/test/java/org/apache/nifi/processors/flume/ExecuteFlumeSinkTest.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.flume;
+
+import java.io.File;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+
+import org.apache.commons.io.filefilter.HiddenFileFilter;
+import org.apache.flume.sink.NullSink;
+import org.apache.flume.source.AvroSource;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.util.MockProcessContext;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.apache.nifi.util.file.FileUtils;
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ExecuteFlumeSinkTest {
+
+ private static final Logger logger =
+ LoggerFactory.getLogger(ExecuteFlumeSinkTest.class);
+
+ @Rule
+ public final TemporaryFolder temp = new TemporaryFolder();
+
+ @Test
+ public void testValidators() {
+ TestRunner runner = TestRunners.newTestRunner(ExecuteFlumeSink.class);
+ Collection<ValidationResult> results;
+ ProcessContext pc;
+
+ results = new HashSet<>();
+ runner.enqueue(new byte[0]);
+ pc = runner.getProcessContext();
+ if (pc instanceof MockProcessContext) {
+ results = ((MockProcessContext) pc).validate();
+ }
+ Assert.assertEquals(1, results.size());
+ for (ValidationResult vr : results) {
+ logger.debug(vr.toString());
+ Assert.assertTrue(vr.toString().contains("is invalid because Sink Type is required"));
+ }
+
+ // non-existent class
+ results = new HashSet<>();
+ runner.setProperty(ExecuteFlumeSink.SINK_TYPE, "invalid.class.name");
+ runner.enqueue(new byte[0]);
+ pc = runner.getProcessContext();
+ if (pc instanceof MockProcessContext) {
+ results = ((MockProcessContext) pc).validate();
+ }
+ Assert.assertEquals(1, results.size());
+ for (ValidationResult vr : results) {
+ logger.debug(vr.toString());
+ Assert.assertTrue(vr.toString().contains("is invalid because unable to load sink"));
+ }
+
+ // class doesn't implement Sink
+ results = new HashSet<>();
+ runner.setProperty(ExecuteFlumeSink.SINK_TYPE, AvroSource.class.getName());
+ runner.enqueue(new byte[0]);
+ pc = runner.getProcessContext();
+ if (pc instanceof MockProcessContext) {
+ results = ((MockProcessContext) pc).validate();
+ }
+ Assert.assertEquals(1, results.size());
+ for (ValidationResult vr : results) {
+ logger.debug(vr.toString());
+ Assert.assertTrue(vr.toString().contains("is invalid because unable to create sink"));
+ }
+
+ results = new HashSet<>();
+ runner.setProperty(ExecuteFlumeSink.SINK_TYPE, NullSink.class.getName());
+ runner.enqueue(new byte[0]);
+ pc = runner.getProcessContext();
+ if (pc instanceof MockProcessContext) {
+ results = ((MockProcessContext) pc).validate();
+ }
+ Assert.assertEquals(0, results.size());
+ }
+
+
+ @Test
+ public void testNullSink() throws IOException {
+ TestRunner runner = TestRunners.newTestRunner(ExecuteFlumeSink.class);
+ runner.setProperty(ExecuteFlumeSink.SINK_TYPE, NullSink.class.getName());
+ try (InputStream inputStream = getClass().getResourceAsStream("/testdata/records.txt")) {
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(CoreAttributes.FILENAME.key(), "records.txt");
+ runner.enqueue(inputStream, attributes);
+ runner.run();
+ }
+ }
+
+ @Test
+ public void testBatchSize() throws IOException {
+ TestRunner runner = TestRunners.newTestRunner(ExecuteFlumeSink.class);
+ runner.setProperty(ExecuteFlumeSink.SINK_TYPE, NullSink.class.getName());
+ runner.setProperty(ExecuteFlumeSink.FLUME_CONFIG,
+ "tier1.sinks.sink-1.batchSize = 1000\n");
+ for (int i = 0; i < 100000; i++) {
+ runner.enqueue(String.valueOf(i).getBytes());
+ }
+ runner.run(100);
+ }
+
+ @Test
+ @Ignore("Does not work on Windows")
+ public void testHdfsSink() throws IOException {
+ File destDir = temp.newFolder("hdfs");
+
+ TestRunner runner = TestRunners.newTestRunner(ExecuteFlumeSink.class);
+ runner.setProperty(ExecuteFlumeSink.SINK_TYPE, "hdfs");
+ runner.setProperty(ExecuteFlumeSink.FLUME_CONFIG,
+ "tier1.sinks.sink-1.hdfs.path = " + destDir.toURI().toString() + "\n" +
+ "tier1.sinks.sink-1.hdfs.fileType = DataStream\n" +
+ "tier1.sinks.sink-1.hdfs.serializer = TEXT\n" +
+ "tier1.sinks.sink-1.serializer.appendNewline = false"
+ );
+ try (InputStream inputStream = getClass().getResourceAsStream("/testdata/records.txt")) {
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(CoreAttributes.FILENAME.key(), "records.txt");
+ runner.enqueue(inputStream, attributes);
+ runner.run();
+ }
+
+ File[] files = destDir.listFiles((FilenameFilter)HiddenFileFilter.VISIBLE);
+ assertEquals("Unexpected number of destination files.", 1, files.length);
+ File dst = files[0];
+ byte[] expectedMd5;
+ try (InputStream md5Stream = getClass().getResourceAsStream("/testdata/records.txt")) {
+ expectedMd5 = FileUtils.computeMd5Digest(md5Stream);
+ }
+ byte[] actualMd5 = FileUtils.computeMd5Digest(dst);
+ Assert.assertArrayEquals("Destination file doesn't match source data", expectedMd5, actualMd5);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/test/java/org/apache/nifi/processors/flume/ExecuteFlumeSourceTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/test/java/org/apache/nifi/processors/flume/ExecuteFlumeSourceTest.java b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/test/java/org/apache/nifi/processors/flume/ExecuteFlumeSourceTest.java
new file mode 100644
index 0000000..2f40e5c
--- /dev/null
+++ b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/test/java/org/apache/nifi/processors/flume/ExecuteFlumeSourceTest.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.flume;
+
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.flume.sink.NullSink;
+import org.apache.flume.source.AvroSource;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.MockProcessContext;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.apache.nifi.util.file.FileUtils;
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ExecuteFlumeSourceTest {
+
+ private static final Logger logger = LoggerFactory.getLogger(ExecuteFlumeSourceTest.class);
+
+ @Rule
+ public final TemporaryFolder temp = new TemporaryFolder();
+
+ @Test
+ public void testValidators() {
+ TestRunner runner = TestRunners.newTestRunner(ExecuteFlumeSource.class);
+ Collection<ValidationResult> results;
+ ProcessContext pc;
+
+ results = new HashSet<>();
+ runner.enqueue(new byte[0]);
+ pc = runner.getProcessContext();
+ if (pc instanceof MockProcessContext) {
+ results = ((MockProcessContext) pc).validate();
+ }
+ Assert.assertEquals(1, results.size());
+ for (ValidationResult vr : results) {
+ logger.debug(vr.toString());
+ Assert.assertTrue(vr.toString().contains("is invalid because Source Type is required"));
+ }
+
+ // non-existent class
+ results = new HashSet<>();
+ runner.setProperty(ExecuteFlumeSource.SOURCE_TYPE, "invalid.class.name");
+ runner.enqueue(new byte[0]);
+ pc = runner.getProcessContext();
+ if (pc instanceof MockProcessContext) {
+ results = ((MockProcessContext) pc).validate();
+ }
+ Assert.assertEquals(1, results.size());
+ for (ValidationResult vr : results) {
+ logger.debug(vr.toString());
+ Assert.assertTrue(vr.toString().contains("is invalid because unable to load source"));
+ }
+
+ // class doesn't implement Source
+ results = new HashSet<>();
+ runner.setProperty(ExecuteFlumeSource.SOURCE_TYPE, NullSink.class.getName());
+ runner.enqueue(new byte[0]);
+ pc = runner.getProcessContext();
+ if (pc instanceof MockProcessContext) {
+ results = ((MockProcessContext) pc).validate();
+ }
+ Assert.assertEquals(1, results.size());
+ for (ValidationResult vr : results) {
+ logger.debug(vr.toString());
+ Assert.assertTrue(vr.toString().contains("is invalid because unable to create source"));
+ }
+
+ results = new HashSet<>();
+ runner.setProperty(ExecuteFlumeSource.SOURCE_TYPE, AvroSource.class.getName());
+ runner.enqueue(new byte[0]);
+ pc = runner.getProcessContext();
+ if (pc instanceof MockProcessContext) {
+ results = ((MockProcessContext) pc).validate();
+ }
+ Assert.assertEquals(0, results.size());
+ }
+
+ @Test
+ public void testSequenceSource() {
+ TestRunner runner = TestRunners.newTestRunner(ExecuteFlumeSource.class);
+ runner.setProperty(ExecuteFlumeSource.SOURCE_TYPE, "seq");
+ runner.run();
+ List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ExecuteFlumeSource.SUCCESS);
+ Assert.assertEquals(1, flowFiles.size());
+ for (MockFlowFile flowFile : flowFiles) {
+ logger.debug(flowFile.toString());
+ Assert.assertEquals(1, flowFile.getSize());
+ }
+ }
+
+ @Test
+ @Ignore("Does not work on Windows")
+ public void testSourceWithConfig() throws IOException {
+ File spoolDirectory = temp.newFolder("spooldir");
+ File dst = new File(spoolDirectory, "records.txt");
+ FileUtils.copyFile(getClass().getResourceAsStream("/testdata/records.txt"), dst, true, false);
+
+ TestRunner runner = TestRunners.newTestRunner(ExecuteFlumeSource.class);
+ runner.setProperty(ExecuteFlumeSource.SOURCE_TYPE, "spooldir");
+ runner.setProperty(ExecuteFlumeSink.FLUME_CONFIG,
+ "tier1.sources.src-1.spoolDir = " + spoolDirectory.getAbsolutePath());
+ runner.run(1, false, true);
+ // Because the spool directory source is an event driven source, it may take some time for flow files to get
+ // produced. I'm willing to wait up to 5 seconds, but will bail out early if possible. If it takes longer than
+ // that then there is likely a bug.
+ int numWaits = 10;
+ while (runner.getFlowFilesForRelationship(ExecuteFlumeSource.SUCCESS).size() < 4 && --numWaits > 0) {
+ try {
+ TimeUnit.MILLISECONDS.sleep(500);
+ } catch (InterruptedException ex) {
+ logger.warn("Sleep interrupted");
+ }
+ }
+ runner.shutdown();
+ runner.assertTransferCount(ExecuteFlumeSource.SUCCESS, 4);
+ int i = 1;
+ for (MockFlowFile flowFile : runner.getFlowFilesForRelationship(ExecuteFlumeSource.SUCCESS)) {
+ flowFile.assertContentEquals("record " + i);
+ i++;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/test/resources/core-site-broken.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/test/resources/core-site-broken.xml b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/test/resources/core-site-broken.xml
new file mode 100644
index 0000000..e06a193
--- /dev/null
+++ b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/test/resources/core-site-broken.xml
@@ -0,0 +1,25 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+ <property>
+ <name>fs.default.name</name>
+ <value>hdfs://localhost:65535</value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/test/resources/core-site.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/test/resources/core-site.xml b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/test/resources/core-site.xml
new file mode 100644
index 0000000..849854b
--- /dev/null
+++ b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/test/resources/core-site.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+ <property>
+ <name>fs.defaultFS</name>
+ <!--
+ Hadoop doesn't support a chroot style operation for the
+ local filesystem so there's no benefit to setting this
+ to a directory other than '/'
+ -->
+ <value>file:///</value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/test/resources/log4j.properties b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/test/resources/log4j.properties
new file mode 100644
index 0000000..cc58727
--- /dev/null
+++ b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/test/resources/log4j.properties
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+log4j.rootLogger=INFO, CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x \u2013 %m%n
+
+log4j.logger.org.apache.flume = DEBUG
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/test/resources/simplelogger.properties
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/test/resources/simplelogger.properties b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/test/resources/simplelogger.properties
new file mode 100644
index 0000000..e3d4fc1
--- /dev/null
+++ b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/test/resources/simplelogger.properties
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+org.slf4j.simpleLogger.logFile=System.out
+org.slf4j.simpleLogger.defaultLogLevel=info
+org.slf4j.simpleLogger.showDateTime=true
+org.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd HH:mm:ss.SSS
+org.slf4j.simpleLogger.levelInBrackets=true
+org.slf4j.simpleLogger.log.org.apache.nifi.processors.flume=debug
+org.slf4j.simpleLogger.log.org.apache.flume=debug
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/test/resources/testdata/records.txt
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/test/resources/testdata/records.txt b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/test/resources/testdata/records.txt
new file mode 100644
index 0000000..5a809ee
--- /dev/null
+++ b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/test/resources/testdata/records.txt
@@ -0,0 +1,4 @@
+record 1
+record 2
+record 3
+record 4
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-flume-bundle/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-flume-bundle/pom.xml b/nifi-nar-bundles/nifi-flume-bundle/pom.xml
new file mode 100644
index 0000000..56ca257
--- /dev/null
+++ b/nifi-nar-bundles/nifi-flume-bundle/pom.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-nar-bundles</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>nifi-flume-bundle</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ <packaging>pom</packaging>
+ <description>A bundle of processors that run Flume sources/sinks</description>
+ <modules>
+ <module>nifi-flume-processors</module>
+ <module>nifi-flume-nar</module>
+ </modules>
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-flume-processors</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework-nar/.gitignore
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework-nar/.gitignore b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework-nar/.gitignore
new file mode 100755
index 0000000..ea8c4bf
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework-nar/.gitignore
@@ -0,0 +1 @@
+/target
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework-nar/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework-nar/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework-nar/pom.xml
new file mode 100644
index 0000000..dbf9296
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework-nar/pom.xml
@@ -0,0 +1,62 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-framework-bundle</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>nifi-framework-nar</artifactId>
+ <packaging>nar</packaging>
+ <description>NiFi: Framework Nar</description>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-jetty</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-cluster-authorization-provider</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-file-authorization-provider</artifactId>
+ </dependency>
+
+ <!-- mark these nifi artifacts as provided since it is included in the lib -->
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-runtime</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-nar-utils</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-properties</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework-nar/src/main/resources/META-INF/LICENSE
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework-nar/src/main/resources/META-INF/LICENSE b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework-nar/src/main/resources/META-INF/LICENSE
new file mode 100644
index 0000000..be85dcb
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework-nar/src/main/resources/META-INF/LICENSE
@@ -0,0 +1,264 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+APACHE NIFI SUBCOMPONENTS:
+
+The Apache NiFi project contains subcomponents with separate copyright
+notices and license terms. Your use of the source code for the these
+subcomponents is subject to the terms and conditions of the following
+licenses.
+
+ The binary distribution of this product bundles 'Antlr 3' which is available
+ under a "3-clause BSD" license. For details see http://www.antlr3.org/license.html
+
+ Copyright (c) 2010 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+
+ Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+ Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+ Neither the name of the author nor the names of its contributors may be used
+ to endorse or promote products derived from this software without specific
+ prior written permission.
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ THE POSSIBILITY OF SUCH DAMAGE.
+
+ The binary distribution of this product bundles 'Bouncy Castle JDK 1.6 Provider'
+ under an MIT style license.
+
+ Copyright (c) 2000 - 2013 The Legion of the Bouncy Castle Inc. (http://www.bouncycastle.org)
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+
+
+
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework-nar/src/main/resources/META-INF/NOTICE
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework-nar/src/main/resources/META-INF/NOTICE b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework-nar/src/main/resources/META-INF/NOTICE
new file mode 100644
index 0000000..5703d10
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework-nar/src/main/resources/META-INF/NOTICE
@@ -0,0 +1,137 @@
+nifi-framework-nar
+Copyright 2014-2015 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+******************
+Apache Software License v2
+******************
+
+The following binary components are provided under the Apache Software License v2
+
+ (ASLv2) Apache Commons IO
+ The following NOTICE information applies:
+ Apache Commons IO
+ Copyright 2002-2012 The Apache Software Foundation
+
+ (ASLv2) Apache Commons Net
+ The following NOTICE information applies:
+ Apache Commons Net
+ Copyright 2001-2013 The Apache Software Foundation
+
+ (ASLv2) Apache Commons Collections
+ The following NOTICE information applies:
+ Apache Commons Collections
+ Copyright 2001-2013 The Apache Software Foundation
+
+ (ASLv2) Apache Commons Compress
+ The following NOTICE information applies:
+ Apache Commons Compress
+ Copyright 2002-2014 The Apache Software Foundation
+
+ The files in the package org.apache.commons.compress.archivers.sevenz
+ were derived from the LZMA SDK, version 9.20 (C/ and CPP/7zip/),
+ which has been placed in the public domain:
+
+ "LZMA SDK is placed in the public domain." (http://www.7-zip.org/sdk.html)
+
+ (ASLv2) Jettison
+ The following NOTICE information applies:
+ Copyright 2006 Envoi Solutions LLC
+
+ (ASLv2) Jasypt
+ The following NOTICE information applies:
+ Copyright (c) 2007-2010, The JASYPT team (http://www.jasypt.org)
+
+ (ASLv2) Apache Commons Codec
+ The following NOTICE information applies:
+ Apache Commons Codec
+ Copyright 2002-2014 The Apache Software Foundation
+
+ src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java
+ contains test data from http://aspell.net/test/orig/batch0.tab.
+ Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org)
+
+ ===============================================================================
+
+ The content of package org.apache.commons.codec.language.bm has been translated
+ from the original php source code available at http://stevemorse.org/phoneticinfo.htm
+ with permission from the original authors.
+ Original source copyright:
+ Copyright (c) 2008 Alexander Beider & Stephen P. Morse.
+
+ (ASLv2) Apache HttpComponents
+ The following NOTICE information applies:
+ Apache HttpClient
+ Copyright 1999-2014 The Apache Software Foundation
+
+ Apache HttpCore
+ Copyright 2005-2014 The Apache Software Foundation
+
+ This project contains annotations derived from JCIP-ANNOTATIONS
+ Copyright (c) 2005 Brian Goetz and Tim Peierls. See http://www.jcip.net
+
+ (ASLv2) Apache Commons Logging
+ The following NOTICE information applies:
+ Apache Commons Logging
+ Copyright 2003-2013 The Apache Software Foundation
+
+ (ASLv2) Apache Commons Lang
+ The following NOTICE information applies:
+ Apache Commons Lang
+ Copyright 2001-2015 The Apache Software Foundation
+
+ This product includes software from the Spring Framework,
+ under the Apache License 2.0 (see: StringUtils.containsWhitespace())
+
+ (ASLv2) Spring Framework
+ The following NOTICE information applies:
+ Spring Framework 4.1.6.RELEASE
+ Copyright (c) 2002-2015 Pivotal, Inc.
+
+ (ASLv2) Swagger Core
+ The following NOTICE information applies:
+ Swagger Core 1.5.3-M1
+ Copyright 2015 Reverb Technologies, Inc.
+
+************************
+Common Development and Distribution License 1.1
+************************
+
+The following binary components are provided under the Common Development and Distribution License 1.1. See project link for details.
+
+ (CDDL 1.1) (GPL2 w/ CPE) jersey-client (com.sun.jersey:jersey-client:jar:1.19 - https://jersey.java.net/jersey-client/)
+ (CDDL 1.1) (GPL2 w/ CPE) jersey-core (com.sun.jersey:jersey-core:jar:1.19 - https://jersey.java.net/jersey-core/)
+ (CDDL 1.1) (GPL2 w/ CPE) jersey-spring (com.sun.jersey:jersey-spring:jar:1.19 - https://jersey.java.net/jersey-spring/)
+ (CDDL 1.1) (GPL2 w/ CPE) jersey-servlet (com.sun.jersey:jersey-servlet:jar:1.19 - https://jersey.java.net/jersey-servlet/)
+ (CDDL 1.1) (GPL2 w/ CPE) jersey-multipart (com.sun.jersey:jersey-multipart:jar:1.19 - https://jersey.java.net/jersey-multipart/)
+ (CDDL 1.1) (GPL2 w/ CPE) jersey-server (com.sun.jersey:jersey-server:jar:1.19 - https://jersey.java.net/jersey-server/)
+ (CDDL 1.1) (GPL2 w/ CPE) jersey-json (com.sun.jersey:jersey-json:jar:1.19 - https://jersey.java.net/jersey-json/)
+ (CDDL 1.1) (GPL2 w/ CPE) Old JAXB Runtime (com.sun.xml.bind:jaxb-impl:jar:2.2.3-1 - http://jaxb.java.net/)
+ (CDDL 1.1) (GPL2 w/ CPE) Java Architecture For XML Binding (javax.xml.bind:jaxb-api:jar:2.2.2 - https://jaxb.dev.java.net/)
+ (CDDL 1.1) (GPL2 w/ CPE) MIME Streaming Extension (org.jvnet.mimepull:mimepull:jar:1.9.3 - http://mimepull.java.net)
+ (CDDL 1.1) (GPL2 w/ CPE) JavaMail API (compat) (javax.mail:mail:jar:1.4.7 - http://kenai.com/projects/javamail/mail)
+
+************************
+Common Development and Distribution License 1.0
+************************
+
+The following binary components are provided under the Common Development and Distribution License 1.0. See project link for details.
+
+ (CDDL 1.0) (GPL3) Streaming API For XML (javax.xml.stream:stax-api:jar:1.0-2 - no url provided)
+ (CDDL 1.0) JavaBeans Activation Framework (JAF) (javax.activation:activation:jar:1.1 - http://java.sun.com/products/javabeans/jaf/index.jsp)
+ (CDDL 1.0) JSR311 API (javax.ws.rs:jsr311-api:jar:1.1.1 - https://jsr311.dev.java.net)
+
+************************
+Eclipse Public License 1.0
+************************
+
+The following binary components are provided under the Eclipse Public License 1.0. See project link for details.
+
+ (EPL 1.0) AspectJ Weaver (org.aspectj:aspectjweaver:jar:1.8.5 - http://www.aspectj.org)
+ (EPL 1.0)(MPL 2.0) H2 Database (com.h2database:h2:jar:1.3.176 - http://www.h2database.com/html/license.html)
+ (EPL 1.0)(LGPL 2.1) Logback Classic (ch.qos.logback:logback-classic:jar:1.1.3 - http://logback.qos.ch/)
+ (EPL 1.0)(LGPL 2.1) Logback Core (ch.qos.logback:logback-core:jar:1.1.3 - http://logback.qos.ch/)
+
+
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/.gitignore
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/.gitignore b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/.gitignore
new file mode 100755
index 0000000..ea8c4bf
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/.gitignore
@@ -0,0 +1 @@
+/target
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/pom.xml
new file mode 100644
index 0000000..269aada
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/pom.xml
@@ -0,0 +1,121 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-framework</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>nifi-administration</artifactId>
+ <build>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ <resource>
+ <directory>src/main/xsd</directory>
+ </resource>
+ </resources>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>jaxb2-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>current</id>
+ <goals>
+ <goal>xjc</goal>
+ </goals>
+ <configuration>
+ <packageName>org.apache.nifi.authorization.generated</packageName>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <configuration>
+ <excludes>**/authorization/generated/*.java,</excludes>
+ </configuration>
+ </plugin>
+
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-user-actions</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-framework-core-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-nar-utils</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-utils</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-properties</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-security-utils</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.h2database</groupId>
+ <artifactId>h2</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-beans</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-context</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-aop</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.aspectj</groupId>
+ <artifactId>aspectjweaver</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-collections4</artifactId>
+ </dependency>
+ </dependencies>
+</project>
[45/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/processor/Relationship.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/Relationship.java b/nifi-api/src/main/java/org/apache/nifi/processor/Relationship.java
new file mode 100644
index 0000000..d9f13be
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/processor/Relationship.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor;
+
+/**
+ * An immutable object for holding information about a type of relationship.
+ *
+ */
+public final class Relationship implements Comparable<Relationship> {
+
+ public static final Relationship SELF = new Relationship.Builder().build();
+ public static final Relationship ANONYMOUS = new Relationship.Builder().name("").build();
+
+ /**
+ * The proper name of the relationship. Determines the relationship
+ * 'identity'
+ */
+ private final String name;
+ /**
+ * A user displayable description of the purpose of this relationship.
+ */
+ private final String description;
+
+ /**
+ * The hash code, which is computed in the constructor because it is hashed
+ * very frequently and the hash code is constant
+ */
+ private final int hashCode;
+
+ protected Relationship(final Builder builder) {
+ this.name = builder.name == null ? null : builder.name.intern();
+ this.description = builder.description;
+ this.hashCode = 301 + this.name.hashCode(); // compute only once, since it gets called a bunch and will never change
+ }
+
+ @Override
+ public int compareTo(final Relationship o) {
+ if (o == null) {
+ return -1;
+ }
+ final String thisName = getName();
+ final String thatName = o.getName();
+ if (thisName == null && thatName == null) {
+ return 0;
+ }
+ if (thisName == null) {
+ return 1;
+ }
+ if (thatName == null) {
+ return -1;
+ }
+
+ return thisName.compareTo(thatName);
+ }
+
+ public static final class Builder {
+
+ private String name = "";
+ private String description = "";
+
+ public Builder name(final String name) {
+ if (null != name) {
+ this.name = name;
+ }
+ return this;
+ }
+
+ public Builder description(final String description) {
+ if (null != description) {
+ this.description = description;
+ }
+ return this;
+ }
+
+ public Relationship build() {
+ return new Relationship(this);
+ }
+ }
+
+ public String getName() {
+ return this.name;
+ }
+
+ public String getDescription() {
+ return this.description;
+ }
+
+ @Override
+ public boolean equals(final Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (!(other instanceof Relationship)) {
+ return false;
+ }
+ if (this == other) {
+ return true;
+ }
+ Relationship desc = (Relationship) other;
+ return this.name.equals(desc.name);
+ }
+
+ @Override
+ public int hashCode() {
+ return hashCode;
+ }
+
+ @Override
+ public String toString() {
+ return this.name;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/processor/SchedulingContext.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/SchedulingContext.java b/nifi-api/src/main/java/org/apache/nifi/processor/SchedulingContext.java
new file mode 100644
index 0000000..0cfc0be
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/processor/SchedulingContext.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor;
+
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.PropertyValue;
+import org.apache.nifi.controller.ControllerServiceLookup;
+
+public interface SchedulingContext extends ProcessContext {
+
+ /**
+ * <p>
+ * Indicates to the framework that the Controller Service with the given
+ * identifier will be used by this Processor. This will prevent any Data
+ * Flow Manager from disabling the Controller Service while this Processor
+ * is still running.
+ * </p>
+ *
+ * <p>
+ * Generally, a Controller Service is accessed by indicating in a
+ * {@link PropertyDescriptor} that the PropertyDescriptor identifies a
+ * Controller Service via the
+ * {@link PropertyDescriptor.Builder#identifiesControllerService(Class) identifiesControllerService(Class)}
+ * method and then calling
+ * {@link ProcessContext#getProperty(PropertyDescriptor)}.{@link PropertyValue#asControllerService(Class) asControllerService(Class)}.
+ * In this case, it is not necessary to lease the Controller Service, as the
+ * Framework will handle this.
+ * </p>
+ *
+ * <p>
+ * There are, however, cases in which a Controller Service must be accessed
+ * in a different way, via the {@link ControllerServiceLookup} (accessed via
+ * {@link ProcessContext#getControllerServiceLookup()}). In this case, the
+ * Controller Service that is obtained from the ControllerServiceLookup can
+ * be disabled by a Data Flow Manager while it is still in use by a
+ * Processor, causing IllegalStateException to be thrown whenever the
+ * Processor attempts to interact with the service. This method provides a
+ * mechanism by which a Processor is able to indicate that the Controller
+ * Service with the given identifier should not be disabled while this
+ * Processor is running.
+ * </p>
+ *
+ * <p>
+ * For any Controller Service that is leased by calling this method, the
+ * lease will automatically be terminated, allowing the Controller Service
+ * to be disabled, whenever the Processor is stopped.
+ * </p>
+ *
+ * @param identifier
+ *
+ * @throws IllegalArgumentException if no Controller Service exists with the
+ * given identifier
+ */
+ void leaseControllerService(String identifier);
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/processor/annotation/CapabilityDescription.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/annotation/CapabilityDescription.java b/nifi-api/src/main/java/org/apache/nifi/processor/annotation/CapabilityDescription.java
new file mode 100644
index 0000000..f966f89
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/processor/annotation/CapabilityDescription.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor.annotation;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Annotation that may be placed on a processor allowing for a description to be
+ * provided. This description can be provided to a user in logs, UI, etc.
+ *
+ * @deprecated This Annotation has been replaced by the
+ * {@link org.apache.nifi.annotation.documentation.CapabilityDescription}
+ * annotation.
+ */
+@Documented
+@Target({ElementType.TYPE})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+@Deprecated
+public @interface CapabilityDescription {
+
+ String value();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/processor/annotation/EventDriven.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/annotation/EventDriven.java b/nifi-api/src/main/java/org/apache/nifi/processor/annotation/EventDriven.java
new file mode 100644
index 0000000..0f412ca
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/processor/annotation/EventDriven.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor.annotation;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * <p>
+ * Annotation that may be placed on a Processor that indicates to the framework
+ * that the Processor is eligible to be scheduled to run based on the occurrence
+ * of an "Event" (e.g., when a FlowFile is enqueued in an incoming Connection),
+ * rather than being triggered periodically.
+ * </p>
+ *
+ * <p>
+ * This Annotation should not be used in conjunction with
+ * {@link TriggerSerially} or {@link TriggerWhenEmpty}. If this Annotation is
+ * used with either of these other Annotations, the Processor will not be
+ * eligible to be scheduled in Event-Driven mode.
+ * </p>
+ *
+ * @deprecated This Annotation has been replaced by the
+ * {@link org.apache.nifi.annotation.behavior.EventDriven} annotation.
+ */
+@Documented
+@Target({ElementType.TYPE})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+@Deprecated
+public @interface EventDriven {
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/processor/annotation/OnAdded.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/annotation/OnAdded.java b/nifi-api/src/main/java/org/apache/nifi/processor/annotation/OnAdded.java
new file mode 100644
index 0000000..ff0b75f
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/processor/annotation/OnAdded.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor.annotation;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Marker annotation a processor implementation can use to indicate a method
+ * should be called whenever the processor is added to the graph. This method
+ * will be called once for the entire life of a processor instance.
+ *
+ * If any method annotated with this annotation throws, the processor will not
+ * be added to the graph.
+ *
+ * @deprecated This Annotation has been replaced by the
+ * {@link org.apache.nifi.annotation.lifecycle.OnAdded} annotation.
+ */
+@Documented
+@Target({ElementType.METHOD})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+@Deprecated
+public @interface OnAdded {
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/processor/annotation/OnRemoved.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/annotation/OnRemoved.java b/nifi-api/src/main/java/org/apache/nifi/processor/annotation/OnRemoved.java
new file mode 100644
index 0000000..740e9f8
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/processor/annotation/OnRemoved.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor.annotation;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Marker annotation a processor implementation can use to indicate a method
+ * should be called whenever the processor is removed from the graph. This
+ * method will be called once for the entire life of a processor instance unless
+ * an invocation of this method throws any Throwable.
+ *
+ * If any method annotated with this annotation throws, the processor will not
+ * be removed from the graph.
+ *
+ * @deprecated This Annotation has been replaced by the
+ * {@link org.apache.nifi.annotation.lifecycle.OnRemoved} annotation.
+ */
+@Documented
+@Target({ElementType.METHOD})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+@Deprecated
+public @interface OnRemoved {
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/processor/annotation/OnScheduled.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/annotation/OnScheduled.java b/nifi-api/src/main/java/org/apache/nifi/processor/annotation/OnScheduled.java
new file mode 100644
index 0000000..3815de2
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/processor/annotation/OnScheduled.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor.annotation;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Marker annotation a processor implementation can use to indicate a method
+ * should be called whenever the processor is scheduled for processing. This
+ * will be called before any 'onTrigger' calls and will be called once each time
+ * a processor instance is scheduled to run. Methods using this annotation must
+ * take either 0 arguments or a single argument of type
+ * {@link nifi.processor.SchedulingContext SchedulingContext}.
+ *
+ * If any method annotated with this annotation throws, the processor will not
+ * be scheduled to run.
+ *
+ * @deprecated This Annotation has been replaced by the
+ * {@link org.apache.nifi.annotation.lifecycle.OnScheduled} annotation.
+ */
+@Documented
+@Target({ElementType.METHOD})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+@Deprecated
+public @interface OnScheduled {
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/processor/annotation/OnShutdown.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/annotation/OnShutdown.java b/nifi-api/src/main/java/org/apache/nifi/processor/annotation/OnShutdown.java
new file mode 100644
index 0000000..930a9df
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/processor/annotation/OnShutdown.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor.annotation;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Marker annotation a processor implementation can use to indicate a method
+ * should be called whenever the flow is being shutdown. This will be called at
+ * most once for each processor instance in a process lifetime.
+ *
+ * @deprecated This Annotation has been replaced by the
+ * {@link org.apache.nifi.annotation.lifecycle.OnShutdown} annotation.
+ */
+@Documented
+@Target({ElementType.METHOD})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+@Deprecated
+public @interface OnShutdown {
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/processor/annotation/OnStopped.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/annotation/OnStopped.java b/nifi-api/src/main/java/org/apache/nifi/processor/annotation/OnStopped.java
new file mode 100644
index 0000000..4fbaf95
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/processor/annotation/OnStopped.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor.annotation;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * <p>
+ * Marker annotation a processor implementation can use to indicate a method
+ * should be called whenever the processor is no longer scheduled to run.
+ * Methods marked with this annotation will be invoked each time the processor
+ * is stopped and will be invoked only after the last thread has returned from
+ * the <code>onTrigger</code> method.
+ * </p>
+ *
+ * <p>
+ * This means that the thread executing in this method will be the only thread
+ * executing in any part of the Processor. However, since other threads may
+ * later execute other parts of the code, member variables must still be
+ * protected appropriately. However, access to multiple variables need not be
+ * atomic.
+ * </p>
+ *
+ * <p>
+ * To indicate that a method should be called immediately when a processor is no
+ * longer scheduled to run, see the {@link OnUnscheduled} annotation.
+ * </p>
+ *
+ * @deprecated This Annotation has been replaced by the
+ * {@link org.apache.nifi.annotation.lifecycle.OnStopped} annotation.
+ */
+@Documented
+@Target({ElementType.METHOD})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+@Deprecated
+public @interface OnStopped {
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/processor/annotation/OnUnscheduled.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/annotation/OnUnscheduled.java b/nifi-api/src/main/java/org/apache/nifi/processor/annotation/OnUnscheduled.java
new file mode 100644
index 0000000..7b4c1f6
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/processor/annotation/OnUnscheduled.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor.annotation;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * <p>
+ * Marker annotation a processor implementation can use to indicate a method
+ * should be called whenever the processor is no longer scheduled to run.
+ * Methods marked with this annotation will be invoked each time the framework
+ * is notified to stop scheduling the processor. This method is invoked as other
+ * threads are potentially running. To invoke a method after all threads have
+ * finished processing, see the {@link OnStopped} annotation.
+ * </p>
+ *
+ * If any method annotated with this annotation throws, the processor will not
+ * be scheduled to run.
+ *
+ * @deprecated This Annotation has been replaced by the
+ * {@link org.apache.nifi.annotation.lifecycle.OnUnscheduled} annotation.
+ */
+@Documented
+@Target({ElementType.METHOD})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+@Deprecated
+public @interface OnUnscheduled {
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/processor/annotation/SideEffectFree.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/annotation/SideEffectFree.java b/nifi-api/src/main/java/org/apache/nifi/processor/annotation/SideEffectFree.java
new file mode 100644
index 0000000..6c65caa
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/processor/annotation/SideEffectFree.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor.annotation;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Marker annotation a processor implementation can use to indicate that its
+ * operations on flow files can be safely repeated across process sessions. If a
+ * processor has this annotation and it allows the framework to manage session
+ * commit and rollback then the framework may elect to cascade a
+ * <code>ProcessSession</code> given to this processor's onTrigger method to the
+ * onTrigger method of another processor. It can do this knowing that if
+ * something fails along a series of processors using this same session that it
+ * can all be safely rolled back without any ill effects on external services
+ * which could not be rolled back and thus all the processes could be safely
+ * repeated (implied idempotent behavior).
+ *
+ * @deprecated This Annotation has been replaced by the
+ * {@link org.apache.nifi.annotation.behavior.SideEffectFree} annotation.
+ */
+@Documented
+@Target({ElementType.TYPE})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+@Deprecated
+public @interface SideEffectFree {
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/processor/annotation/SupportsBatching.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/annotation/SupportsBatching.java b/nifi-api/src/main/java/org/apache/nifi/processor/annotation/SupportsBatching.java
new file mode 100644
index 0000000..5b3d1aa
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/processor/annotation/SupportsBatching.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor.annotation;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Marker annotation a Processor implementation can use to indicate that users
+ * should be able to supply a Batch Duration for the Processor. If a Processor
+ * uses this annotation, it is allowing the Framework to batch
+ * {@link nifi.processor.ProcessSession ProcessSession}s' commits, as well as
+ * allowing the Framework to return the same ProcessSession multiple times from
+ * subsequent calls to
+ * {@link nifi.processor.ProcessSessionFactory ProcessSessionFactory}.{@link nifi.processor.ProcessSessionFactory#createSession() createSession()}.
+ *
+ * When this Annotation is used, it is important to note that calls to
+ * {@link nifi.processor.ProcessSession#commit() ProcessSession.commit()} may
+ * not provide a guarantee that the data has been safely stored in NiFi's
+ * Content Repository or FlowFile Repository. Therefore, it is not appropriate,
+ * for instance, to use this annotation if the Processor will call
+ * ProcessSession.commit() to ensure data is persisted before deleting the data
+ * from a remote source.
+ *
+ * @deprecated This Annotation has been replaced by the
+ * {@link org.apache.nifi.annotation.behavior.SupportsBatching} annotation.
+ */
+@Documented
+@Target({ElementType.TYPE})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+@Deprecated
+public @interface SupportsBatching {
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/processor/annotation/Tags.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/annotation/Tags.java b/nifi-api/src/main/java/org/apache/nifi/processor/annotation/Tags.java
new file mode 100644
index 0000000..81428d7
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/processor/annotation/Tags.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor.annotation;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Annotation that can be applied to a {@link Processor} in order to associate
+ * tags with the processor. These tags do not affect the {@link Processor} in
+ * any way but serve as additional documentation and can be used to sort/filter
+ * Processors.
+ *
+ * @deprecated This Annotation has been replaced by the
+ * {@link org.apache.nifi.annotation.documentation.Tags} annotation.
+ */
+@Documented
+@Target({ElementType.TYPE})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+@Deprecated
+public @interface Tags {
+
+ public String[] value();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/processor/annotation/TriggerSerially.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/annotation/TriggerSerially.java b/nifi-api/src/main/java/org/apache/nifi/processor/annotation/TriggerSerially.java
new file mode 100644
index 0000000..6e80cef
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/processor/annotation/TriggerSerially.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor.annotation;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Marker annotation a processor implementation can use to indicate that the
+ * processor is not thread safe for concurrent execution of its onTrigger()
+ * method. By default processors are assumed to be thread safe for concurrent
+ * execution.
+ *
+ * @deprecated This Annotation has been replaced by the
+ * {@link org.apache.nifi.annotation.behavior.TriggerSerially} annotation.
+ */
+@Documented
+@Target({ElementType.TYPE})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+@Deprecated
+public @interface TriggerSerially {
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/processor/annotation/TriggerWhenAnyDestinationAvailable.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/annotation/TriggerWhenAnyDestinationAvailable.java b/nifi-api/src/main/java/org/apache/nifi/processor/annotation/TriggerWhenAnyDestinationAvailable.java
new file mode 100644
index 0000000..6e01f6c
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/processor/annotation/TriggerWhenAnyDestinationAvailable.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor.annotation;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Marker annotation a processor implementation can use to indicate that the
+ * processor is to be triggered if any of its destinations has available space
+ * for incoming FlowFiles. By default processors are triggered only when all
+ * destinations report that they have available space.
+ *
+ * @deprecated This Annotation has been replaced by the
+ * {@link org.apache.nifi.annotation.behavior.TriggerWhenAnyDestinationAvailable}
+ * annotation.
+ */
+@Documented
+@Target({ElementType.TYPE})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+@Deprecated
+public @interface TriggerWhenAnyDestinationAvailable {
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/processor/annotation/TriggerWhenEmpty.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/annotation/TriggerWhenEmpty.java b/nifi-api/src/main/java/org/apache/nifi/processor/annotation/TriggerWhenEmpty.java
new file mode 100644
index 0000000..d068a9e
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/processor/annotation/TriggerWhenEmpty.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor.annotation;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Marker annotation a processor implementation can use to indicate that the
+ * processor should still be triggered even when it has no data in its work
+ * queue. By default, processors which have no non-self incoming edges will be
+ * triggered even if there is no work in its queue. However, processors that
+ * have non-self incoming edges will only be triggered if they have work in
+ * their queue or they present this annotation.
+ *
+ * @deprecated This Annotation has been replaced by the
+ * {@link org.apache.nifi.annotation.behavior.TriggerWhenEmpty} annotation.
+ */
+@Documented
+@Target({ElementType.TYPE})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+@Deprecated
+public @interface TriggerWhenEmpty {
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/processor/exception/FlowFileAccessException.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/exception/FlowFileAccessException.java b/nifi-api/src/main/java/org/apache/nifi/processor/exception/FlowFileAccessException.java
new file mode 100644
index 0000000..c7e9c22
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/processor/exception/FlowFileAccessException.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor.exception;
+
+/**
+ * Indicates an issue occurred while accessing the content of a FlowFile, such
+ * as an IOException.
+ *
+ */
+public class FlowFileAccessException extends RuntimeException {
+
+ private static final long serialVersionUID = 1L;
+
+ public FlowFileAccessException(final String message) {
+ super(message);
+ }
+
+ public FlowFileAccessException(final String message, final Throwable cause) {
+ super(message, cause);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/processor/exception/FlowFileHandlingException.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/exception/FlowFileHandlingException.java b/nifi-api/src/main/java/org/apache/nifi/processor/exception/FlowFileHandlingException.java
new file mode 100644
index 0000000..d5bf62d
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/processor/exception/FlowFileHandlingException.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor.exception;
+
+/**
+ * Thrown when a flow file is referenced that is not part of the appropriate
+ * session or is not properly accounted for by a transfer or removal within a
+ * session. In any event this exception indicates a logic or programming error
+ * within the processor interacting with the offending session.
+ *
+ */
+public class FlowFileHandlingException extends ProcessException {
+
+ private static final long serialVersionUID = 1L;
+
+ public FlowFileHandlingException(final String message) {
+ super(message);
+ }
+
+ public FlowFileHandlingException(final String message, final Throwable cause) {
+ super(message, cause);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/processor/exception/MissingFlowFileException.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/exception/MissingFlowFileException.java b/nifi-api/src/main/java/org/apache/nifi/processor/exception/MissingFlowFileException.java
new file mode 100644
index 0000000..700afd5
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/processor/exception/MissingFlowFileException.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor.exception;
+
+/**
+ * Thrown to indicate that the content for some FlowFile could not be found.
+ * This indicates that the data is gone and likely cannot be restored and any
+ * information about the FlowFile should be discarded entirely. This likely
+ * indicates some influence from an external process outside the control of the
+ * framework and it is not something any processor or the framework can recover
+ * so it must discard the object.
+ *
+ */
+public class MissingFlowFileException extends RuntimeException {
+
+ private static final long serialVersionUID = 1L;
+
+ public MissingFlowFileException(final String message, final Throwable cause) {
+ super(message, cause);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/processor/exception/ProcessException.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/exception/ProcessException.java b/nifi-api/src/main/java/org/apache/nifi/processor/exception/ProcessException.java
new file mode 100644
index 0000000..f162b14
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/processor/exception/ProcessException.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor.exception;
+
+/**
+ * Indicates an issue occurred in user code while processing a FlowFile.
+ *
+ */
+public class ProcessException extends RuntimeException {
+
+ private static final long serialVersionUID = 1L;
+
+ public ProcessException() {
+ super();
+ }
+
+ public ProcessException(final String message) {
+ super(message);
+ }
+
+ public ProcessException(final String message, final Throwable cause) {
+ super(message, cause);
+ }
+
+ public ProcessException(final Throwable cause) {
+ super(cause);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/processor/io/InputStreamCallback.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/io/InputStreamCallback.java b/nifi-api/src/main/java/org/apache/nifi/processor/io/InputStreamCallback.java
new file mode 100644
index 0000000..6873f2e
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/processor/io/InputStreamCallback.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor.io;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ *
+ */
+public interface InputStreamCallback {
+
+ /**
+ * Provides a managed input stream for use. The input stream is
+ * automatically opened and closed though it is ok to close the stream
+ * manually.
+ *
+ * @param in the stream to read bytes from
+ * @throws IOException if issues reading from the underlying stream
+ */
+ void process(InputStream in) throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/processor/io/OutputStreamCallback.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/io/OutputStreamCallback.java b/nifi-api/src/main/java/org/apache/nifi/processor/io/OutputStreamCallback.java
new file mode 100644
index 0000000..86fe1a7
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/processor/io/OutputStreamCallback.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor.io;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ *
+ */
+public interface OutputStreamCallback {
+
+ /**
+ * Provides a managed output stream for use. The input stream is
+ * automatically opened and closed though it is ok to close the stream
+ * manually - and quite important if any streams wrapping these streams open
+ * resources which should be cleared.
+ *
+ * @param out the stream to write bytes to
+ * @throws IOException if issues writing to output stream
+ */
+ void process(OutputStream out) throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/processor/io/StreamCallback.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/io/StreamCallback.java b/nifi-api/src/main/java/org/apache/nifi/processor/io/StreamCallback.java
new file mode 100644
index 0000000..54f0e3b
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/processor/io/StreamCallback.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor.io;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+public interface StreamCallback {
+
+ /**
+ * Provides a managed output stream for use. The input stream is
+ * automatically opened and closed though it is ok to close the stream
+ * manually - and quite important if any streams wrapping these streams open
+ * resources which should be cleared.
+ *
+ * @param in the stream to read bytes from
+ * @param out the stream to write bytes to
+ * @throws IOException if issues occur reading or writing the underlying streams
+ */
+ void process(InputStream in, OutputStream out) throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/provenance/ProvenanceEventBuilder.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/provenance/ProvenanceEventBuilder.java b/nifi-api/src/main/java/org/apache/nifi/provenance/ProvenanceEventBuilder.java
new file mode 100644
index 0000000..0ffccd5
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/provenance/ProvenanceEventBuilder.java
@@ -0,0 +1,308 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.Processor;
+import org.apache.nifi.processor.Relationship;
+
+public interface ProvenanceEventBuilder {
+
+ /**
+ * Sets the type of {@link ProvenanceEventRecord}
+ *
+ * @param eventType of the event
+ * @return the builder
+ */
+ ProvenanceEventBuilder setEventType(ProvenanceEventType eventType);
+
+ /**
+ * Populates the values of the Event being built from the values in the
+ * given event
+ *
+ * @param event the event from which to populate the Builders values
+ * @return the builder
+ */
+ ProvenanceEventBuilder fromEvent(ProvenanceEventRecord event);
+
+ /**
+ * Sets the date and time at which the FlowFile entered the flow
+ *
+ * @param entryDate of the flow file
+ * @return the builder
+ */
+ ProvenanceEventBuilder setFlowFileEntryDate(long entryDate);
+
+ /**
+ * Sets the Lineage Identifiers. This is a set of all FlowFile UUID's that
+ * were involved in making this event occur.
+ *
+ * @param lineageIdentifiers of the flowfiles in this event
+ * @return the builder
+ */
+ ProvenanceEventBuilder setLineageIdentifiers(Set<String> lineageIdentifiers);
+
+ /**
+ * Sets the Content Claim that the FlowFile was previously associated with
+ * before this event occurred.
+ *
+ * @param container for previous content
+ * @param section for previous content
+ * @param identifier for previous content
+ * @param offset for previous content
+ * @param size for previous content
+ * @return the builder
+ */
+ ProvenanceEventBuilder setPreviousContentClaim(String container, String section, String identifier, Long offset, long size);
+
+ /**
+ * Sets the Content Claim that the FlowFile is associated with as a result
+ * of this event
+ *
+ * @param container for resulting content
+ * @param section for resulting content
+ * @param identifier for resulting content
+ * @param offset for resulting content
+ * @param size for resulting content
+ * @return the builder
+ */
+ ProvenanceEventBuilder setCurrentContentClaim(String container, String section, String identifier, Long offset, long size);
+
+ /**
+ * Sets the identifier of the FlowFile Queue from which the FlowFile was
+ * pulled
+ *
+ * @param identifier of the source queue
+ * @return the builder
+ */
+ ProvenanceEventBuilder setSourceQueueIdentifier(String identifier);
+
+ /**
+ * Sets the attributes that existed on the FlowFile before this event
+ * occurred and any attributes that were added or updated as a result of
+ * this event.
+ *
+ * @param previousAttributes Map of all attributes before the event occurred
+ * @param updatedAttributes Map containing all attributes that were added or
+ * updated. If any entry has a value of <code>null</code>, that attribute is
+ * considered removed
+ *
+ * @return the builder
+ */
+ ProvenanceEventBuilder setAttributes(Map<String, String> previousAttributes, Map<String, String> updatedAttributes);
+
+ /**
+ * Sets the UUID to associate with the FlowFile
+ *
+ * @param uuid of the flowfile
+ * @return the builder
+ */
+ ProvenanceEventBuilder setFlowFileUUID(String uuid);
+
+ /**
+ * Sets the time at which the Provenance Event took place
+ *
+ * @param eventTime time of the event
+ * @return the builder
+ */
+ ProvenanceEventBuilder setEventTime(long eventTime);
+
+ /**
+ * Sets the amount of time that was required in order to perform the
+ * function referred to by this event
+ *
+ * @param millis of the event
+ * @return the builder
+ */
+ ProvenanceEventBuilder setEventDuration(long millis);
+
+ /**
+ * Sets the time at which the FlowFile's lineage began
+ *
+ * @param startDate start date of the event
+ * @return the builder
+ */
+ ProvenanceEventBuilder setLineageStartDate(long startDate);
+
+ /**
+ * Sets the unique identifier of the NiFi Component (such as a
+ * {@link Processor}) that is generating the Event
+ *
+ * @param componentId that produced the event
+ * @return the builder
+ */
+ ProvenanceEventBuilder setComponentId(String componentId);
+
+ /**
+ * Sets the type of the Component that is generating the Event. For
+ * {@link Processor}s, this is the Simple Class Name of the Processor.
+ *
+ * @param componentType of the component that made the event
+ * @return the builder
+ */
+ ProvenanceEventBuilder setComponentType(String componentType);
+
+ /**
+ * Sets the identifier that is used by the remote system to refer to the
+ * FlowFile for which this Event is being created.
+ *
+ * This is valid only for Event Types {@link ProvenanceEventType#RECEIVE}
+ * and {@link ProvenanceEventType#SEND} and will be ignored for any other
+ * event types.
+ *
+ * @param sourceSystemFlowFileIdentifier identifier the remote system used
+ * @return the builder
+ */
+ ProvenanceEventBuilder setSourceSystemFlowFileIdentifier(String sourceSystemFlowFileIdentifier);
+
+ /**
+ * Sets the Transit URI that is used for the Event. This is a URI that
+ * provides information about the System and Protocol information over which
+ * the transfer occurred. The intent of this field is such that both the
+ * sender and the receiver can publish the events to an external
+ * Enterprise-wide system that is then able to correlate the SEND and
+ * RECEIVE events.
+ *
+ * This is valid only for Event Types {@link ProvenanceEventType#RECEIVE}
+ * and {@link ProvenanceEventType#SEND} and will be ignored for any other
+ * event types.
+ *
+ * @param transitUri of the event
+ * @return the builder
+ */
+ ProvenanceEventBuilder setTransitUri(String transitUri);
+
+ /**
+ * Adds the given FlowFile as a parent for Events of type {@link ProvenanceEventType#SPAWN},
+ * {@link ProvenanceEventType#FORK}, {@link ProvenanceEventType#JOIN}, and
+ * {@link ProvenanceEventType#CLONE}
+ *
+ * This is valid only for {@link ProvenanceEventType#SPAWN},
+ * {@link ProvenanceEventType#FORK}, {@link ProvenanceEventType#JOIN}, and
+ * {@link ProvenanceEventType#CLONE} events and will be ignored for any
+ * other event types.
+ *
+ * @param parent flowfile that this event is derived from
+ * @return the builder
+ */
+ ProvenanceEventBuilder addParentFlowFile(FlowFile parent);
+
+ /**
+ * Removes the given FlowFile as a parent for Events of type {@link ProvenanceEventType#SPAWN},
+ * {@link ProvenanceEventType#FORK}, {@link ProvenanceEventType#JOIN}, and
+ * {@link ProvenanceEventType#CLONE}
+ *
+ * This is valid only for {@link ProvenanceEventType#SPAWN},
+ * {@link ProvenanceEventType#FORK}, {@link ProvenanceEventType#JOIN}, and
+ * {@link ProvenanceEventType#CLONE} events and will be ignored for any
+ * other event types.
+ *
+ * @param parent previous parent of this event
+ * @return the builder
+ */
+ ProvenanceEventBuilder removeParentFlowFile(FlowFile parent);
+
+ /**
+ * Adds the given FlowFile as a child for Events of type {@link ProvenanceEventType#SPAWN},
+ * {@link ProvenanceEventType#FORK}, {@link ProvenanceEventType#JOIN}, and
+ * {@link ProvenanceEventType#CLONE}
+ *
+ * This is valid only for {@link ProvenanceEventType#SPAWN},
+ * {@link ProvenanceEventType#FORK}, {@link ProvenanceEventType#JOIN}, and
+ * {@link ProvenanceEventType#CLONE} events and will be ignored for any
+ * other event types.
+ *
+ * @param child the child to add
+ * @return the builder
+ */
+ ProvenanceEventBuilder addChildFlowFile(FlowFile child);
+
+ /**
+ * Removes the given FlowFile as a child for Events of type {@link ProvenanceEventType#SPAWN},
+ * {@link ProvenanceEventType#FORK}, {@link ProvenanceEventType#JOIN}, and
+ * {@link ProvenanceEventType#CLONE}
+ *
+ * This is valid only for {@link ProvenanceEventType#SPAWN},
+ * {@link ProvenanceEventType#FORK}, {@link ProvenanceEventType#JOIN}, and
+ * {@link ProvenanceEventType#CLONE} events and will be ignored for any
+ * other event types.
+ *
+ * @param child to remove
+ * @return the builder
+ */
+ ProvenanceEventBuilder removeChildFlowFile(FlowFile child);
+
+ /**
+ * Associates the given identifier with the FlowFile for which this Event is
+ * created. This provides a linkage between the given FlowFile and alternate
+ * identifier. This information can be useful if published to an external,
+ * enterprise-wide Provenance tracking system that is able to associate the
+ * data between different processes or services.
+ *
+ * This is valid only for {@link ProvenanceEventType#ADDINFO} events and
+ * will be ignored for any other event types.
+ *
+ * @param alternateIdentifierUri another identifier of the flowfile this event is for
+ * @return the builder
+ */
+ ProvenanceEventBuilder setAlternateIdentifierUri(String alternateIdentifierUri);
+
+ /**
+ * Sets the details for this event. This is a free-form String that can
+ * contain any information that is relevant to this event.
+ *
+ * @param details a description of the event
+ * @return the builder
+ */
+ ProvenanceEventBuilder setDetails(String details);
+
+ /**
+ * Sets the to which the FlowFile was routed for
+ * {@link ProvenanceEventType#ROUTE} events. This is valid only for
+ * {@link ProvenanceEventType#ROUTE} events and will be ignored for any
+ * other event types.
+ *
+ * @param relationship to which flowfiles in this event were routed
+ * @return the builder
+ */
+ ProvenanceEventBuilder setRelationship(Relationship relationship);
+
+ /**
+ * Populates the builder with as much information as it can from the given
+ * FlowFile
+ *
+ * @param flowFile to source attributes for this event from
+ * @return the builder
+ */
+ ProvenanceEventBuilder fromFlowFile(FlowFile flowFile);
+
+ /**
+ * Builds the Provenance Event. It is possible that calling
+ * {@link ProvenanceEventRecord#getEventId()} on the
+ * {@link ProvenanceEventRecord} that is returned will yield
+ * <code>-1</code>. This is because the implementation of the Event may
+ * depend on the {@link ProvevenanceEventRepository} to generate the unique
+ * identifier.
+ *
+ * @return the event
+ */
+ ProvenanceEventRecord build();
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/provenance/ProvenanceEventRecord.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/provenance/ProvenanceEventRecord.java b/nifi-api/src/main/java/org/apache/nifi/provenance/ProvenanceEventRecord.java
new file mode 100644
index 0000000..dc251b3
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/provenance/ProvenanceEventRecord.java
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Describes an event that happened to a FlowFile.
+ */
+public interface ProvenanceEventRecord {
+
+ /**
+ * @return a unique ID for this Provenance Event. Depending on the
+ * implementation, the Event ID may be set to -1 until the event has been
+ * added to the {@link ProvenanceEventRepository}
+ */
+ long getEventId();
+
+ /**
+ * @return the time at which this Provenance Event was created, as the
+ * number of milliseconds since epoch
+ */
+ long getEventTime();
+
+ /**
+ * @return the EntryDate of the FlowFile to which this Event is associated
+ */
+ long getFlowFileEntryDate();
+
+ /**
+ * @return the time at which the lineage began
+ */
+ long getLineageStartDate();
+
+ /**
+ * @return the set of all lineage identifiers that are associated with the
+ * FlowFile for which this Event was created
+ */
+ Set<String> getLineageIdentifiers();
+
+ /**
+ * @return the size of the FlowFile to which this Event is associated
+ */
+ long getFileSize();
+
+ /**
+ * @return the previous size of the FlowFile to which this Event is
+ * associated, if the FlowFile previously had content and its size was
+ * known; otherwise, returns <code>null</code>
+ */
+ Long getPreviousFileSize();
+
+ /**
+ * @return the amount of time in milliseconds that elapsed while performing
+ * this event. If not populated, the value -1 will be returned
+ */
+ long getEventDuration();
+
+ /**
+ * @return the type of this Provenance Event
+ */
+ ProvenanceEventType getEventType();
+
+ /**
+ * @return all FlowFile attributes that were associated with the FlowFile at
+ * the time that this ProvenanceEvent was created
+ */
+ Map<String, String> getAttributes();
+
+ /**
+ * @return all FlowFile attributes that existed on the FlowFile before this
+ * event occurred
+ */
+ Map<String, String> getPreviousAttributes();
+
+ /**
+ * @return all FlowFile attributes that were updated as a result of this
+ * event
+ */
+ Map<String, String> getUpdatedAttributes();
+
+ /**
+ * @return the ID of the Processor/component that created this Provenance
+ * Event
+ */
+ String getComponentId();
+
+ /**
+ * @return the fully-qualified Class Name of the Processor/component that
+ * created this Provenance Event
+ */
+ String getComponentType();
+
+ /**
+ * @return a URI that provides information about the System and Protocol
+ * information over which the transfer occurred. The intent of this field is
+ * such that both the sender and the receiver can publish the events to an
+ * external Enterprise-wide system that is then able to correlate the SEND
+ * and RECEIVE events.
+ */
+ String getTransitUri();
+
+ /**
+ * Since the receiving system will usually refer to the data using a
+ * different identifier than the source system, this information is used to
+ * correlate the receive system's FlowFile with the sending system's data
+ *
+ * @return the UUID that the Source System used to refer to this data; this
+ * is applicable only when the {@link ProvenanceEventType} is of type
+ * {@link ProvenanceEventType#RECEIVE RECEIVE}
+ */
+ String getSourceSystemFlowFileIdentifier();
+
+ /**
+ * @return the UUID of the FlowFile with which this Event is associated
+ */
+ String getFlowFileUuid();
+
+ /**
+ * @return the UUID's of all Parent FlowFiles. This is applicable only when
+ * the {@link ProvenanceEventType} is of type
+ * {@link ProvenanceEventType#SPAWN SPAWN}
+ */
+ List<String> getParentUuids();
+
+ /**
+ * @return the UUID's of all Child FlowFiles. This is applicable only when
+ * the {@link ProvenanceEventType} is of type
+ * {@link ProvenanceEventType#SPAWN SPAWN}
+ */
+ List<String> getChildUuids();
+
+ /**
+ * @return the Alternate Identifier associated with the FlowFile with which
+ * this Event is associated. This is applicable only when the
+ * {@link ProvenanceEventType} is of type
+ * {@link ProvenanceEventType#ADDINFO}
+ */
+ String getAlternateIdentifierUri();
+
+ /**
+ * @return the details for this record, if any were supplied. Otherwise,
+ * returns <code>null</code>
+ */
+ String getDetails();
+
+ /**
+ * @return the relationship to which this record was routed if the event
+ * type is {@link ProvenanceEventType#ROUTE}. The relationship is applicable
+ * only to this type
+ */
+ String getRelationship();
+
+ /**
+ * @return the identifier of the queue from which the FlowFile was taken, if
+ * any. If the FlowFile is created as a result of this event (in this case,
+ * the Event Type is one of null null null null null null null null {@link ProvenanceEventType#CREATE}, {@link ProvenanceEventType#RECEIVE},
+ * {@link ProvenanceEventType#FORK}, {@link ProvenanceEventType#JOIN}, or
+ * {@link ProvenanceEventType#CLONE}), or if the queue identifier is
+ * unknown, then this method will return <code>null</code>
+ *
+ */
+ String getSourceQueueIdentifier();
+
+ /**
+ * @return the Section for the Content Claim that this Event refers to, if
+ * any; otherwise, returns <code>null</code>
+ *
+ */
+ String getContentClaimSection();
+
+ /**
+ * @return the Section for the Content Claim that the FlowFile previously
+ * referenced, if any; otherwise, returns <code>null</code>
+ *
+ */
+ String getPreviousContentClaimSection();
+
+ /**
+ * @return the Container for the Content Claim that this Event refers to, if
+ * any; otherwise, returns <code>null</code>
+ *
+ */
+ String getContentClaimContainer();
+
+ /**
+ * @return the Container for the Content Claim that the FlowFile previously
+ * referenced, if any; otherwise, returns <code>null</code>
+ */
+ String getPreviousContentClaimContainer();
+
+ /**
+ * @return the Identifier for the Content Claim that this Event refers to,
+ * if any; otherwise, returns <code>null</code>
+ *
+ */
+ String getContentClaimIdentifier();
+
+ /**
+ * @return the Identifier for the Content Claim that the FlowFile previously
+ * referenced, if any; otherwise, returns <code>null</code>
+ *
+ */
+ String getPreviousContentClaimIdentifier();
+
+ /**
+ * @return the offset into the Content Claim at which the FlowFile's content
+ * begins, if any; otherwise, returns <code>null</code>
+ *
+ */
+ Long getContentClaimOffset();
+
+ /**
+ * @return the offset into the Content Claim at which the FlowFile's
+ * previous content began, if any; otherwise, returns <code>null</code>
+ *
+ */
+ Long getPreviousContentClaimOffset();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/provenance/ProvenanceEventRepository.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/provenance/ProvenanceEventRepository.java b/nifi-api/src/main/java/org/apache/nifi/provenance/ProvenanceEventRepository.java
new file mode 100644
index 0000000..25563b7
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/provenance/ProvenanceEventRepository.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.nifi.events.EventReporter;
+import org.apache.nifi.provenance.lineage.ComputeLineageSubmission;
+import org.apache.nifi.provenance.search.Query;
+import org.apache.nifi.provenance.search.QuerySubmission;
+import org.apache.nifi.provenance.search.SearchableField;
+
+/**
+ * This Repository houses Provenance Events. The repository is responsible for
+ * managing the life-cycle of the events, providing access to the events that it
+ * has stored, and providing query capabilities against the events.
+ *
+ */
+public interface ProvenanceEventRepository {
+
+ /**
+ * Performs any initialization needed. This should be called only by the
+ * framework.
+ *
+ * @param eventReporter to report to
+ * @throws java.io.IOException if unable to initialize
+ */
+ void initialize(EventReporter eventReporter) throws IOException;
+
+ /**
+ * Returns a {@link ProvenanceEventBuilder} that is capable of building
+ * {@link ProvenanceEventRecord}s
+ *
+ * @return builder
+ */
+ ProvenanceEventBuilder eventBuilder();
+
+ /**
+ * Adds the given event to the repository and returns a new event for which
+ * the event id has been populated. Depending on the implementation, the
+ * returned event may or may not be the same event given
+ *
+ * @param event to register
+ */
+ void registerEvent(ProvenanceEventRecord event);
+
+ /**
+ * Adds the given events to the repository.
+ *
+ * <p>
+ * This interface makes no assumptions about whether or not the registration
+ * of the Collection are atomic. This detail is implementation-specific.
+ * </p>
+ *
+ * @param events to register
+ */
+ void registerEvents(Iterable<ProvenanceEventRecord> events);
+
+ /**
+ * Returns a List of all <code>ProvenanceEventRecord</code>s in the
+ * repository starting with the given ID. The first ID in the repository
+ * will always be 0 or higher.
+ *
+ * @param firstRecordId id of the first record to retrieve
+ * @param maxRecords maximum number of records to retrieve
+ * @return records
+ * @throws java.io.IOException if error reading from repository
+ */
+ List<ProvenanceEventRecord> getEvents(long firstRecordId, final int maxRecords) throws IOException;
+
+ /**
+ * @return the largest ID of any event that is queryable in the repository.
+ * If no queryable events exists, returns null
+ */
+ Long getMaxEventId();
+
+ /**
+ * Submits an asynchronous request to process the given query, returning an
+ * identifier that can be used to fetch the results at a later time
+ *
+ * @param query to submit
+ * @return an identifier that can be used to fetch the results at a later
+ * time
+ */
+ QuerySubmission submitQuery(Query query);
+
+ /**
+ * @param queryIdentifier of the query
+ *
+ * @return the QueryResult associated with the given identifier, if the
+ * query has finished processing. If the query has not yet finished running,
+ * returns <code>null</code>
+ */
+ QuerySubmission retrieveQuerySubmission(String queryIdentifier);
+
+ /**
+ * Submits a Lineage Computation to be completed and returns the
+ * AsynchronousLineageResult that indicates the status of the request and
+ * the results, if the computation is complete.
+ *
+ * @param flowFileUuid the UUID of the FlowFile for which the Lineage should
+ * be calculated
+ * @return a {@link ComputeLineageSubmission} object that can be used to
+ * check if the computing is complete and if so get the results
+ */
+ ComputeLineageSubmission submitLineageComputation(String flowFileUuid);
+
+ /**
+ * @param lineageIdentifier identifier of lineage to compute
+ * @return the {@link ComputeLineageSubmission} associated with the given
+ * identifier
+ */
+ ComputeLineageSubmission retrieveLineageSubmission(String lineageIdentifier);
+
+ /**
+ * @param id to lookup
+ * @return the Provenance Event Record with the given ID, if it exists, or
+ * {@code null} otherwise
+ * @throws IOException if failure while retrieving event
+ */
+ ProvenanceEventRecord getEvent(long id) throws IOException;
+
+ /**
+ * Submits a request to expand the parents of the event with the given id
+ *
+ * @param eventId the one-up id of the Event to expand
+ * @return a submission which can be checked for status
+ *
+ * @throws IllegalArgumentException if the given identifier identifies a
+ * Provenance Event that has a Type that is not expandable or if the
+ * identifier cannot be found
+ */
+ ComputeLineageSubmission submitExpandParents(long eventId);
+
+ /**
+ * Submits a request to expand the children of the event with the given id
+ *
+ * @param eventId the one-up id of the Event
+ * @return a submission which can be checked for status
+ *
+ * @throws IllegalArgumentException if the given identifier identifies a
+ * Provenance Event that has a Type that is not expandable or if the
+ * identifier cannot be found
+ */
+ ComputeLineageSubmission submitExpandChildren(long eventId);
+
+ /**
+ * Closes the repository, freeing any resources
+ *
+ * @throws IOException if failure closing repository
+ */
+ void close() throws IOException;
+
+ /**
+ * @return a list of all fields that can be searched via the
+ * {@link #submitQuery(nifi.provenance.search.Query)} method
+ */
+ List<SearchableField> getSearchableFields();
+
+ /**
+ * @return a list of all FlowFile attributes that can be searched via the
+ * {@link #submitQuery(nifi.provenance.search.Query)} method
+ */
+ List<SearchableField> getSearchableAttributes();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/provenance/ProvenanceEventType.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/provenance/ProvenanceEventType.java b/nifi-api/src/main/java/org/apache/nifi/provenance/ProvenanceEventType.java
new file mode 100644
index 0000000..f2638fe
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/provenance/ProvenanceEventType.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+public enum ProvenanceEventType {
+
+ /**
+ * A CREATE event is used when a FlowFile is generated from data that was
+ * not received from a remote system or external process
+ */
+ CREATE,
+ /**
+ * Indicates a provenance event for receiving data from an external process
+ */
+ RECEIVE,
+ /**
+ * Indicates a provenance event for sending data to an external process
+ */
+ SEND,
+ /**
+ * Indicates a provenance event for the conclusion of an object's life for
+ * some reason other than object expiration
+ */
+ DROP,
+ /**
+ * Indicates a provenance event for the conclusion of an object's life due
+ * to the fact that the object could not be processed in a timely manner
+ */
+ EXPIRE,
+ /**
+ * FORK is used to indicate that one or more FlowFile was derived from a
+ * parent FlowFile.
+ */
+ FORK,
+ /**
+ * JOIN is used to indicate that a single FlowFile is derived from joining
+ * together multiple parent FlowFiles.
+ */
+ JOIN,
+ /**
+ * CLONE is used to indicate that a FlowFile is an exact duplicate of its
+ * parent FlowFile.
+ */
+ CLONE,
+ /**
+ * CONTENT_MODIFIED is used to indicate that a FlowFile's content was
+ * modified in some way. When using this Event Type, it is advisable to
+ * provide details about how the content is modified.
+ */
+ CONTENT_MODIFIED,
+ /**
+ * ATTRIBUTES_MODIFIED is used to indicate that a FlowFile's attributes were
+ * modified in some way. This event is not needed when another event is
+ * reported at the same time, as the other event will already contain all
+ * FlowFile attributes.
+ */
+ ATTRIBUTES_MODIFIED,
+ /**
+ * ROUTE is used to show that a FlowFile was routed to a specified
+ * {@link nifi.processor.Relationship Relationship} and should provide
+ * information about why the FlowFile was routed to this relationship.
+ */
+ ROUTE,
+ /**
+ * Indicates a provenance event for adding additional information such as a
+ * new linkage to a new URI or UUID
+ */
+ ADDINFO,
+ /**
+ * Indicates a provenance event for replaying a FlowFile. The UUID of the
+ * event will indicate the UUID of the original FlowFile that is being
+ * replayed. The event will contain exactly one Parent UUID that is also the
+ * UUID of the FlowFile that is being replayed and exactly one Child UUID
+ * that is the UUID of the a newly created FlowFile that will be re-queued
+ * for processing.
+ */
+ REPLAY
+}
[16/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/expression-language-guide.adoc
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/expression-language-guide.adoc b/nifi-docs/src/main/asciidoc/expression-language-guide.adoc
new file mode 100644
index 0000000..af0ee02
--- /dev/null
+++ b/nifi-docs/src/main/asciidoc/expression-language-guide.adoc
@@ -0,0 +1,1745 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+Apache NiFi Expression Language Guide
+=====================================
+Apache NiFi Team <de...@nifi.apache.org>
+:homepage: http://nifi.apache.org
+
+[[overview]]
+Overview
+--------
+All data in Apache NiFi is represented by an abstraction called a FlowFile.
+A FlowFile is comprised of two major pieces: content and attributes.
+The content portion of the FlowFile represents the data on which to operate.
+For instance, if a file is picked up from a local file system using the
+GetFile Processor, the contents of the file will become the contents of the
+FlowFile.
+
+The attributes portion of the FlowFile represents information about the data
+itself, or metadata. Attributes are key-value pairs that represent what is
+known about the data as well as information that is useful for routing and
+processing the data appropriately.
+Keeping with the example of a file that is picked up from
+a local file system, the FlowFile would have an attribute called `filename` that
+reflected the name of the file on the file system. Additionally, the FlowFile will
+have a `path` attribute that reflects the directory on the file system that this
+file lived in. The FlowFile will also have an attribute named `uuid`, which is a
+unique identifier for this FlowFile.
+
+However, placing these attributes on a FlowFile do not provide much benefit
+if the user is unable to make use of them. The NiFi Expression Language provides
+the ability to reference these attributes, compare them to other values,
+and manipulate their values.
+
+
+[[structure]]
+Structure of a NiFi Expression
+------------------------------
+
+The NiFi Expression Language always begins with the start delimiter `${` and ends
+with the end delimiter `}`. Between the start and end delimiters is the text of the
+Expression itself. In its most basic form, the Expression can consist of just an
+attribute name. For example, `${filename}` will return the value of the ``filename''
+attribute.
+
+In a slightly more complex example, we can instead return a manipulation of this value.
+We can, for example, return an all upper-case version of the filename by calling the
+`toUpper` function: `${filename:toUpper()}`. In this case, we reference the ``filename''
+attribute and then manipulate this value by using the `toUpper` function. A function call
+consists of 5 elements. First, there is a function call delimiter `:`. Second is the name
+of the function - in this case, ``toUpper''. Next is an open parenthesis (`(`), followed
+by the function arguments. The arguments necessary are dependent upon which function
+is being called. In this example, we are using the `toUpper` function, which does not
+have any arguments, so this element is omitted. Finally, the closing parenthesis (`)`)
+indicates the end of the function call. There are many different functions that are supported
+by the Expression Language to achieve many different goals. Some functions provide String (text)
+manipulation, such as the `toUpper` function. Others, such as the `equals` and `matches` functions,
+provide comparison functionality. Functions also exist for manipulating dates and times and
+for performing mathematical operations. Each of these functions is described below, in the
+<<functions> section, with an explanation of what the function does, the arguments that it
+requires, and the type of information that it returns.
+
+When we perform a function call on an attribute, as above, we refer to the attribute as the
+_subject_ of the function, as the attribute is the entity on which the function is operating.
+We can then chain together multiple function calls, where the return value of the first function
+becomes the subject of the second function and its return value becomes the subject of the third
+function and so on. Continuing with our example, we can chain together multiple functions by using
+the expression `${filename:toUpper():equals('HELLO.TXT')}`. There is no limit to the number of
+functions that can be chained together.
+
+Any FlowFile attribute can be referenced using the Expression Language. However, if the attribute
+name contains a ``special character,'' the attribute name must be escaped by quoting it. The following
+characters are each considered ``special characters'':
+
+- $ (dollar sign)
+- | (pipe)
+- { (open brace)
+- } (close brace)
+- ( (open parenthesis)
+- ) (close parenthesis)
+- [ (open bracket)
+- ] (close bracket)
+- , (comma)
+- : (colon)
+- ; (semicolon)
+- / (forward slash)
+- * (asterisk)
+- ' (single quote)
+- (space)
+- \t (tab)
+- \r (carriage return)
+- \n (new-line)
+
+Additionally, a number is considered a ``special character'' if it is the first character of the attribute name.
+If any of these special characters is present in an attribute is quoted by using either single or double quotes.
+The Expression Language allows single quotes and double quotes to be used interchangeably. For example, the following
+can be used to escape an attribute named ``my attribute'': `${"my attribute"}` or `${'my attribute'}`.
+
+In this example, the value to be returned is the value of the "my attribute" value, if it exists. If that attribute
+does not exist, the Expression Language will then look for a System Environment Variable named "my attribute." If
+unable to find this, it will look for a JVM System Property named "my attribute." Finally, if none of these exists,
+the Expression Language will return a `null` value.
+
+There also exist some functions that expect to have no subject. These functions are invoked simply
+by calling the function at the beginning of the Expression, such as `${hostname()}`. These functions
+can then be changed together, as well. For example, `${hostname():toUpper()}`. Attempting to
+evaluate the function with subject will result in an error. In the <<functions>>
+section below, these functions will clearly indicate in their descriptions that they do not
+require a subject.
+
+Often times, we will need to compare the values of two different attributes to each other.
+We are able to accomplish this by using embedded Expressions. We can, for example, check if
+the ``filename'' attribute is the same as the ``uuid'' attribute: `${filename:equals( ${uuid} )}`.
+Notice here, also, that we have a space between the opening parenthesis for the `equals` method and
+the embedded Expression. This is not necessary and does not affect how the Expression is evaluated
+in any way. Rather, it is intended to make the Expression easier to read. White space is ignored by
+the Expression Language between delimiters. Therefore, we can use the Expression
+`${ filename : equals(${ uuid}) }` or `${filename:equals(${uuid})}` and both Expressions
+mean the same thing. We cannot, however, use `${file name:equals(${uuid})}`, because this results
+in `file` and `name` being interpreted as different tokens, rather than a single token, `filename`.
+
+
+
+[[usage]]
+== Expression Language in the Application
+
+The Expression Language is used heavily throughout the NiFi application for configuring Processor
+properties. Not all Processor properties support the Expression Language, however. Whether or not
+a Property supports the Expression Language is determined by the developer of the Processor when
+the Processor is written. However, the application strives to clearly illustrate for each Property
+whether or not the Expression Language is supported.
+
+In the application, when configuring a Processor property, the User Interface provides an Information
+icon (
+image:iconInfo.png["Info"]
+) next to the name of the Property. Hovering over this icon with the mouse will provide a tooltip that
+provides helpful information about the Property. This information includes a description of the Property,
+the default value (if any), historically configured values (if any), and whether or not this Property
+supports the expression language.
+
+
+[[editor]]
+=== Expression Language Editor
+
+When configuring the value of a Processor property, the NiFi User Interface provides help with the
+Expression Language using the Expression Language editor. Once an Expression is begin by typing `${`,
+the editor begins to highlight parentheses and braces so that the user is easily able to tell which
+opening parenthesis or brace matches which closing parenthesis or brace.
+
+The editor also supplies context-sensitive help by providing a list of all functions that can be used
+at the current cursor position. To activate this feature, press Ctrl+Space on the keyboard. The user
+is also able to type part of a function name and then press Ctrl+Space to see all functions that can
+be used that start with the same prefix. For example, if we type into the editor `${filename:to`
+and then press Ctrl+Space, we are provided a pop-up that lists six different functions: `toDate`,
+`toLower`, `toNumber`, `toRadix`, `toString`, and `toUpper`. We can then continue typing to narrow
+which functions are shown, or we can select one of the functions from the list by double-clicking
+it with the mouse or using the arrow keys to highlight the desired function and pressing Enter.
+
+
+
+[[functions]]
+== Functions
+
+Functions provide a convenient way to manipulate and compare values of attributes. The Expression Language
+provides many different functions to meet the needs of a automated dataflow. Each function takes
+zero or more arguments and returns a single value. These functions can then be chained together to create
+powerful Expressions to evaluate conditions and manipulate values. See <<structure>> for more information
+on how to call and chain functions together.
+
+[[types]]
+=== Data Types
+
+Each argument to a function and each value returned from a function has a specific data type. The Expression
+Language supports four different data types:
+
+- *String*: A String is a sequence of characters that can consist of numbers, letters, white space, and
+ special characters.
+- *Number*: A Number is an integer comprised of one or more digits (`0` through `9`). The Expression Language
+ does not provide support for fractional numbers. Dates and times are represented in the
+ Expression Language as Numbers, representing the number of milliseconds since midnight GMT on January 1, 1970.
+- *Boolean*: A Boolean is one of either `true` or `false`.
+
+All attributes are considered to be of type String.
+
+The Expression Language is generally able to automatically coerce a value of one data type to the appropriate
+data type for a function. However, functions do exist to manually coerce a value into a specific data type.
+See the <<type_cast>> section for more information.
+
+
+
+
+
+
+[[boolean]]
+== Boolean Logic
+
+One of the most powerful features of the Expression Language is the ability to compare an attribute value against
+some other value. This is used often, for example, to configure how a Processor should route data. The following
+functions are used for performing boolean logic, such as comparing two values.
+Each of these functions returns a value of type Boolean.
+
+
+[.function]
+=== isNull
+*Description*: [.description]#The `isNull` function returns `true` if the subject is null, `false` otherwise. This is typically used to determine
+if an attribute exists.#
+
+*Subject Type*: [.subject]#Any#
+
+*Arguments*: No arguments
+
+*Return Type*: [.returnType]#Boolean#
+
+*Examples*: `${filename:isNull()}` returns `true` if the "filename" attribute does not exist.
+ It returns `true` if the attribute exists.
+
+
+
+[.function]
+=== notNull
+*Description*: [.description]#The `notNull` function returns the opposite value of the `isNull` function. That is, it will return `true` if the
+subject exists and `false` otherwise.#
+
+*Subject Type*: [.subject]#Any#
+
+*Arguments*: No arguments
+
+*Return Type*: [.returnType]#Boolean#
+
+*Examples*: `${filename:notNull()}` returns `true` if the "filename" attribute exists. It returns "false" if the attribute
+ does not exist.
+
+
+
+[.function]
+=== isEmpty
+*Description*: [.description]#The `isEmpty` function returns `true` if the Subject is null or contains only white-space
+ (new line, carriage return, space, tab), `false` otherwise.#
+
+*Subject Type*: [.subject]#String#
+
+*Arguments*: No arguments
+
+*Return Type*: [.returnType]#Boolean#
+
+*Examples*: `${filename:isEmpty()}` returns `true` if the "filename" attribute does not exist or contains only
+ white space.
+
+
+
+
+[.function]
+=== equals
+
+[.description]
+*Description*: [.description]#The `equals` function is very widely used and determines if its subject is equal to another String value.
+ Note that the `equals` function performs a direct comparison of two String values. Take care not to confuse this
+ function with the <<matches>> function, which evaluates its subject against a Regular Expression.#
+
+[.subject]
+*Subject Type*: [.subject]#Any#
+
+[.arguments]
+*Arguments*:
+
+ - [.argName]#_value_# : [.argDesc]#The value to compare the Subject to. Must be same type as the Subject.#
+
+[.returnType]
+*Return Type*: [.returnType]#Boolean#
+
+[.examples]
+*Examples*:
+We can check if the filename of a FlowFile is "hello.txt" by using the expression `${filename:equals('hello.txt')}`,
+or we could check if the value of the attribute `hello` is equal to the value of the `filename` attribute:
+`${hello:equals( ${filename} )}`.
+
+
+
+[.function]
+=== equalsIgnoreCase
+*Description*: [.description]#Similar to the `equals` function, the `equalsIgnoreCase` function compares its subject against a String value but returns
+`true` if the two values differ only by case (upper case vs. lower case).#
+
+*Subject Type*: [.subject]#String#
+
+*Arguments*:
+
+ - [.argName]#_value_# : [.argDesc]#The value to compare the Subject to.#
+
+*Return Type*: [.returnType]#Boolean#
+
+*Examples*: `${filename:equalsIgnoreCase('hello.txt')}` will evaluate to `true` if filename is equal to "hello.txt"
+ or "HELLO.TXT" or "HeLLo.TxT".
+
+
+
+
+[.function]
+=== gt
+*Description*: [.description]#The `gt` function is used for numeric comparison and returns `true` if the subject is Greater Than
+ its argument. If either the subject or the argument cannot be coerced into a Number,
+ this function returns `false`.#
+
+*Subject Type*: [.subject]#Number#
+
+*Arguments*:
+
+ - [.argName]#_value_# : [.argDesc]#The number to compare the Subject to.#
+
+*Return Type*: [.returnType]#Boolean#
+
+*Examples*: `${fileSize:gt( 1024 )}` will return `true` if the size of the FlowFile's content is more than 1 kilobyte
+ (1024 bytes). Otherwise, it will return `false`.
+
+
+
+
+[.function]
+=== ge
+*Description*: [.description]#The `ge` function is used for numeric comparison and returns `true` if the subject is Greater Than
+ Or Equal To its argument. If either the subject or the argument cannot be coerced into a Number,
+ this function returns `false`.#
+
+*Subject Type*: [.subject]#Number#
+
+*Arguments*:
+
+ - [.argName]#_value_# : [.argDesc]#The number to compare the Subject to.#
+
+*Return Type*: [.returnType]#Boolean#
+
+*Examples*: `${fileSize:ge( 1024 )}` will return `true` if the size of the FlowFile's content is at least (
+ is greater than or equal to) 1 kilobyte (1024 bytes). Otherwise, it will return `false`.
+
+
+
+[.function]
+=== lt
+*Description*: [.description]#The `lt` function is used for numeric comparison and returns `true` if the subject is Less Than
+ its argument. If either the subject or the argument cannot be coerced into a Number,
+ this function returns `false`.#
+
+*Subject Type*: [.subject]#Number#
+
+*Arguments*:
+
+ - [.argName]#_value_# : [.argDesc]#The number to compare the Subject to.#
+
+*Return Type*: [.returnType]#Boolean#
+
+*Examples*: `${fileSize:lt( 1048576 )}` will return `true` if the size of the FlowFile's content is less than
+ 1 megabyte (1048576 bytes). Otherwise, it will return `false`.
+
+
+
+
+[.function]
+=== le
+*Description*: [.description]#The `le` function is used for numeric comparison and returns `true` if the subject is Less Than
+ Or Equal To its argument. If either the subject or the argument cannot be coerced into a Number,
+ this function returns `false`.#
+
+*Subject Type*: [.subject]#Number#
+
+*Arguments*:
+
+ - [.argName]#_value_# : [.argDesc]#The number to compare the Subject to.#
+
+*Return Type*: [.returnType]#Boolean#
+
+*Examples*: `${fileSize:le( 1048576 )}` will return `true` if the size of the FlowFile's content is at most
+ (less than or equal to) 1 megabyte (1048576 bytes). Otherwise, it will return `false`.
+
+
+
+
+
+
+[.function]
+=== and
+*Description*: [.description]#The `and` function takes as a single argument a Boolean value and returns `true` if both the Subject
+ and the argument are `true`. If either the subject or the argument is `false` or cannot be coerced into a Boolean,
+ the function returns `false`. Typically, this is used with an embedded Expression as the argument.#
+
+*Subject Type*: [.subject]#Boolean#
+
+*Arguments*:
+
+ - [.argName]#_condition_# : [.argDesc]#The right-hand-side of the 'and' Expression#
+
+
+*Return Type*: [.returnType]#Boolean#
+
+*Examples*: We can check if the filename is both all lower-case and has at least 5 characters by using the Expression
+-----------------------------------------------
+${filename:toLower():equals( ${filename} ):and(
+ ${filename:length():ge(5)}
+)}
+-----------------------------------------------
+
+
+
+
+
+[.function]
+=== or
+
+*Description*: [.description]#The `or` function takes as a single argument a Boolean value and returns `true` if either the Subject
+ or the argument is `true`. If both the subject and the argument are `false`, the function returns `false`. If
+ either the Subject or the argument cannot be coerced into a Boolean value, this function will return `false`.#
+
+*Subject Type*: [.subject]#Boolean#
+
+*Arguments*:
+
+ - [.argName]#_condition_# : [.argDesc]#The right-hand-side of the 'and' Expression#
+
+*Return Type*: [.returnType]#Boolean#
+
+*Examples*: The following example will return `true` if either the filename has exactly 5 characters or if
+ the filename is all lower-case.
+----------------------------------------------
+${filename:toLower():equals( ${filename} ):or(
+ ${filename:length():equals(5)}
+)}
+----------------------------------------------
+
+
+
+[.function]
+=== not
+
+[.description]
+*Description*: [.description]#The `not` function returns the negation of the Boolean value of the subject.#
+
+[.subject]
+*Subject Type*: [.subject]#Boolean#
+
+[.arguments]
+*Arguments*: No arguments
+
+[.returnType]
+*Return Type*: [.returnType]#Boolean#
+
+[.examples]
+*Examples*: We can invert the value of another function by using the `not` function, as
+ `${filename:equals('hello.txt'):not()}`. This will return `true` if the filename is NOT equal to
+ "hello.txt" and will return `false` if the filename is "hello.txt."
+
+
+
+
+
+
+
+[[strings]]
+== String Manipulation
+
+Each of the following functions manipulates a String in some way.
+
+
+
+
+[.function]
+=== toUpper
+
+*Description*: [.description]#This function converts the Subject into an all upper-case String. Said another way, it
+ replaces any lowercase letter with the uppercase equivalent.#
+
+*Subject Type*: [.subject]#String#
+
+*Arguments*: No arguments
+
+*Return Type*: [.returnType]#String#
+
+*Examples*: If the "filename" attribute is "abc123.txt", then the Expression `${filename:toUpper()}`
+ will return "ABC123.TXT"
+
+
+
+
+
+[.function]
+=== toLower
+
+*Description*: [.description]#This function converts the Subject into an all lower-case String. Said another way,
+ it replaces any uppercase letter with the lowercase equivalent.#
+
+*Subject Type*: [.subject]#String#
+
+*Arguments*: No arguments
+
+*Return Type*: [.returnType]#String#
+
+*Examples*: If the "filename" attribute is "ABC123.TXT", then the Expression `${filename:toLower()}`
+ will return "abc123.txt"
+
+
+
+
+
+[.function]
+=== trim
+
+*Description*: [.description]#The `trim` function will remove any leading or trailing white space from its subject.#
+
+*Subject Type*: [.subject]#String#
+
+*Arguments*: No arguments
+
+*Return Type*: [.returnType]#String#
+
+*Examples*: If the attribute `attr` has the value " 1 2 3 ", then the Expression `${attr:trim()}` will
+ return the value "1 2 3".
+
+
+
+
+
+[.function]
+=== urlEncode
+
+*Description*: [.description]#Returns a URL-friendly version of the Subject. This is useful, for instance, when using an
+ attribute value to indicate the URL of a website.#
+
+*Subject Type*: [.subject]#String#
+
+*Arguments*: No arguments
+
+*Return Type*: [.returnType]#String#
+
+*Examples*: We can URL-Encode an attribute named "url" by using the Expression `${url:urlEncode()}`. If
+ the value of the "url" attribute is "https://nifi.apache.org/some value with spaces", this
+ Expression will then return "https://nifi.apache.org/some%20value%20with%20spaces".
+
+
+
+
+[.function]
+=== urlDecode
+
+*Description*: [.description]#Converts a URL-friendly version of the Subject into a human-readable form.#
+
+*Subject Type*: [.subject]#String#
+
+*Arguments*: No arguments
+
+*Return Type*: [.returnType]#String#
+
+*Examples*: If we have a URL-Encoded attribute named "url" with the value
+ "https://nifi.apache.org/some%20value%20with%20spaces", then the Expression
+ `${url:urlDecode()}` will return "https://nifi.apache.org/some value with spaces".
+
+
+
+
+
+[.function]
+=== substring
+
+*Description*:
+[.description]#Returns a portion of the Subject, given a _starting index_ and an optional _ending index_.
+ If the _ending index_ is not supplied, it will return the portion of the Subject starting at the given
+ 'start index' and ending at the end of the Subject value.#
+
+[.description]#The _starting index_ and _ending index_ are zero-based. That is, the first character is referenced by using
+ the value `0`, not `1`.#
+
+[.description]#If either the _starting index_ is or the _ending index_ is not a number, this function call will result
+ in an error.#
+
+[.description]#If the _starting index_ is larger than the _ending index_, this function call will result in an error.#
+
+[.description]#If the _starting index_ or the _ending index_ is greater than the length of the Subject or has a value
+ less than 0, this function call will result in an error.#
+
+
+*Subject Type*: [.subject]#String#
+
+*Arguments*:
+
+ - [.argName]#_starting index_# : [.argDesc]#The 0-based index of the first character to capture (inclusive)#
+ - [.argName]#_ending index_# : [.argDesc]#The 0-based index of the last character to capture (exclusive)#
+
+*Return Type*: [.returnType]#String#
+
+*Examples*:
+
+If we have an attribute named "filename" with the value "a brand new filename.txt",
+then the following Expressions will result in the following values:
+
+.Substring Examples
+|================================================================
+| Expression | Value
+| `${filename:substring(0,1)}` | `a`
+| `${filename:substring(2)}` | `brand new filename.txt`
+| `${filename:substring(12)}` | `filename.txt`
+| `${filename:substring( ${filename:length():minus(2)} )}` | `xt`
+|================================================================
+
+
+
+
+[.function]
+=== substringBefore
+
+*Description*: [.description]#Returns a portion of the Subject, starting with the first character of the Subject
+ and ending with the character immediately before the first occurrence of the argument. If
+ the argument is not present in the Subject, the entire Subject will be returned.#
+
+*Subject Type*: [.subject]#String#
+
+*Arguments*:
+
+ - [.argName]#_value_# : [.argDesc]#The String to search for in the Subject#
+
+*Return Type*: [.returnType]#String#
+
+*Examples*: If the "filename" attribute has the value "a brand new filename.txt",
+ then the following Expressions will result in the following values:
+
+.SubstringBefore Examples
+|======================================================================
+| Expression | Value
+| `${filename:substringBefore('.')}` | `a brand new filename`
+| `${filename:substringBefore(' ')}` | `a`
+| `${filename:substringBefore(' n')}` | `a brand`
+| `${filename:sbustringBefore('missing')}` | `a brand new filename.txt`
+|======================================================================
+
+
+
+
+
+[.function]
+=== substringBeforeLast
+
+*Description*: [.description]#Returns a portion of the Subject, starting with the first character of the Subject
+ and ending with the character immediately before the last occurrence of the argument. If
+ the argument is not present in the Subject, the entire Subject will be returned.#
+
+*Subject Type*: [.subject]#String#
+
+*Arguments*:
+
+ - [.argName]#_value_# : [.argDesc]#The String to search for in the Subject#
+
+*Return Type*: [.returnType]#String3
+
+*Examples*: If the "filename" attribute has the value "a brand new filename.txt",
+ then the following Expressions will result in the following values:
+
+.SubstringBeforeLast Examples
+|==========================================================================
+| Expression | Value
+| `${filename:substringBeforeLast('.')}` | `a brand new filename`
+| `${filename:substringBeforeLast(' ')}` | `a brand new`
+| `${filename:substringBeforeLast(' n')}` | `a brand`
+| `${filename:substringBeforeLast('missing')}` | `a brand new filename.txt`
+|==========================================================================
+
+
+
+
+
+
+[.function]
+=== substringAfter
+
+*Description*: [.description]#Returns a portion of the Subject, starting with the character immediately after
+ the first occurrence of the argument and extending to the end of the Subject. If
+ the argument is not present in the Subject, the entire Subject will be returned.#
+
+*Subject Type*: [.subject]#String#
+
+*Arguments*:
+
+ - [.argName]#_value_# : [.argDesc]#The String to search for in the Subject#
+
+*Return Type*: [.returnType]#String#
+
+*Examples*: If the "filename" attribute has the value "a brand new filename.txt",
+ then the following Expressions will result in the following values:
+
+.SubstringAfter Examples
+|======================================================================
+| Expression | Value
+| `${filename:substringAfter('.')}` | `txt`
+| `${filename:substringAfter(' ')}` | `brand new filename.txt`
+| `${filename:substringAfter(' n')}` | `ew filename.txt`
+| `${filename:substringAfter('missing')}` | `a brand new filename.txt`
+|======================================================================
+
+
+
+
+
+[.function]
+=== substringAfterLast
+
+*Description*: [.description]#Returns a portion of the Subject, starting with the character immediately after
+ the last occurrence of the argument and extending to the end of the Subject. If
+ the argument is not present in the Subject, the entire Subject will be returned.#
+
+*Subject Type*: [.subject]#String#
+
+*Arguments*:
+
+ - [.argName]#_value_# : [.argDesc]#The String to search for in the Subject#
+
+*Return Type*: [.returnType]#String#
+
+*Examples*: If the "filename" attribute has the value "a brand new filename.txt",
+ then the following Expressions will result in the following values:
+
+.SubstringAfterLast Examples
+|=========================================================================
+| Expression | Value
+| `${filename:substringAfterLast('.')}` | `txt`
+| `${filename:substringAfterLast(' ')}` | `filename.txt`
+| `${filename:substringAfterLast(' n')}` | `ew filename.txt`
+| `${filename:substringAfterLast('missing')}` | `a brand new filename.txt`
+|=========================================================================
+
+
+
+
+
+
+
+[.function]
+=== append
+
+*Description*: [.description]#The `append` function returns the result of appending the argument to the value of
+ the Subject. If the Subject is null, returns the argument itself.#
+
+*Subject Type*: [.subject]#String#
+
+*Arguments*:
+
+ - [.argName]#_value_# : [.argDesc]#The String to append to the end of the Subject#
+
+*Return Type*: [.returnType]#String#
+
+*Examples*: If the "filename" attribute has the value "a brand new filename.txt", then the Expression
+ `${filename:append('.gz')}` will return "a brand new filename.txt.gz".
+
+
+
+
+
+[.function]
+=== prepend
+
+*Description*: [.description]#The `prepend` function returns the result of prepending the argument to the value of
+ the Subject. If the subject is null, returns the argument itself.#
+
+*Subject Type*: [.subject]#String#
+
+*Arguments*:
+
+ - [.argName]#_value_# : [.argDesc]#The String to prepend to the beginning of the Subject#
+
+
+*Return Type*: [.returnType]#String#
+
+*Examples*: If the "filename" attribute has the value "filename.txt", then the Expression
+ `${filename:prepend('a brand new ')}` will return "a brand new filename.txt".
+
+
+
+
+
+[.function]
+=== replace
+
+*Description*: [.description]#Replaces occurrences of one String within the Subject with another String.#
+
+*Subject Type*: [.subject]#String#
+
+*Arguments*:
+
+ - [.argName]#_Search String_# : [.argDesc]#The String to find within the Subject#
+ - [.argName]#_Replacement_# : [.argDesc]#The value to replace _Search String_ with#
+
+*Return Type*: [.returnType]#String#
+
+*Examples*: If the "filename" attribute has the value "a brand new filename.txt", then the following
+Expressions will provide the following results:
+
+
+
+.Replace Examples
+|===================================================================
+| Expression | Value
+| `${filename:replace('.', '_')}` | `a brand new filename_txt`
+| `${filename:replace(' ', '.')}` | `a.brand.new.filename.txt`
+| `${filename:replace('XYZ', 'ZZZ')}` | `a brand new filename.txt`
+| `${filename:replace('filename', 'book')}` | `a brand new book.txt`
+|===================================================================
+
+
+
+
+
+[.function]
+=== replaceAll
+
+*Description*: [.description]#The `replaceAll` function takes two String arguments: a Regular Expression (NiFi uses the Java Pattern
+ syntax), and a replacement string. The return value is the result of substituting the replacement string for
+ all patterns within the Subject that match the Regular Expression.#
+
+*Subject Type*: [.subject]#String#
+
+*Arguments*:
+
+*Arguments*:
+
+ - [.argName]#_Regex_# : [.argDesc]#he Regular Expression (in Java syntax) to match in the Subject#
+ - [.argName]#_Replacement_# : [.argDesc]#The value to use for replacing matches in the Subject. If the _regular expression_
+ argument uses Capturing Groups, back references are allowed in the _replacement_.#
+
+*Return Type*: [.returnType]#String#
+
+*Examples*: If the "filename" attribute has the value "a brand new filename.txt", then the following
+Expressions will provide the following results:
+
+
+
+.replaceAll Examples
+|=======================================================================================
+| Expression | Value
+| `${filename:replaceAll('\..*', '')}` | `a brand new filename`
+| `${filename:replaceAll('a brand (new)', '$1')}` | `new filename.txt`
+| `${filename:replaceAll('XYZ', 'ZZZ')}` | `a brand new filename.txt`
+| `${filename:replaceAll('brand (new)', 'somewhat $1')}` | `a somewhat new filename.txt`
+|=======================================================================================
+
+
+
+
+
+
+[.function]
+=== replaceNull
+
+*Description*: [.description]#The `replaceNull` function returns the argument if the Subject is null. Otherwise,
+ returns the Subject.#
+
+*Subject Type*: [.subject]#Any#
+
+*Arguments*:
+
+ - [.argName]#_Replacement_# : [.argDesc]#The value to return if the Subject is null.#
+
+*Return Type*: [.returnType]#Type of Subject if Subject is not null; else, type of Argument#
+
+*Examples*: If the attribute "filename" has the value "a brand new filename.txt" and the attribute
+ "hello" does not exist, then the Expression `${filename:replaceNull('abc')}` will return
+ "a brand new filename.txt", while `${hello:replaceNull('abc')}` will return "abc".
+
+
+
+
+[.function]
+=== replaceEmpty
+
+*Description*: [.description]#The `replaceEmpty` function returns the argument if the Subject is null or
+ if the Subject consists only of white space (new line, carriage return, tab, space). Otherwise,
+ returns the Subject.#
+
+*Subject Type*: [.subject]#String#
+
+*Arguments*:
+
+ - [.argName]#_Replacement_# : [.argDesc]#The value to return if the Subject is null or empty.#
+
+*Return Type*: [.returnType]#String#
+
+*Examples*: If the attribute "filename" has the value "a brand new filename.txt" and the attribute
+ "hello" has the value " ", then the Expression `${filename:replaceEmpty('abc')}` will return
+ "a brand new filename.txt", while `${hello:replaceEmpty('abc')}` will return "abc".
+
+
+
+
+[.function]
+=== length
+
+*Description*: [.description]#Returns the length of the Subject#
+
+*Subject Type*: [.subject]#String#
+
+*Arguments*: No arguments
+
+*Return Type*: [.returnType]#Number#
+
+*Examples*: If the attribute "filename" has a value of "a brand new filename.txt" and the attribute
+ "hello" does not exist, then the Expression `${filename:length()}` will return 24. `${hello:length()}`
+ will return 0.
+
+
+
+
+
+
+
+
+[[searching]]
+== Searching
+
+Each of the following functions is used to search its subject for some value.
+
+
+[.function]
+=== startsWith
+
+*Description*: [.description]#Returns `true` if the Subject starts with the String provided as the argument,
+ `false` otherwise.#
+
+*Subject Type*: [.subject]#String#
+
+*Arguments*:
+
+ - [.argName]#_value_# : [.argDesc]#The value to search for#
+
+
+*Return Type*: [.returnType]#Boolean#
+
+*Examples*: If the "filename" attribute has the value "a brand new filename.txt", then the Expression
+ `${filename:startsWith('a brand')}` will return `true`. `${filename:startsWith('A BRAND')}` will
+ return `false`. `${filename:toUpper():startsWith('A BRAND')}` returns `true`.
+
+
+
+
+
+[.function]
+=== endsWith
+
+*Description*: [.description]#Returns `true` if the Subject ends with the String provided as the argument,
+ `false` otherwise.#
+
+*Subject Type*: [.subject]#String#
+
+*Arguments*:
+
+ - [.argName]#_value_# : [.argDesc]#The value to search for#
+
+*Return Type*: [.returnType]#Boolean#
+
+*Examples*: If the "filename" attribute has the value "a brand new filename.txt", then the Expression
+ `${filename:endsWith('txt')}` will return `true`. `${filename:endsWith('TXT')}` will
+ return `false`. `${filename:toUpper():endsWith('TXT')}` returns `true`.
+
+
+
+
+
+[.function]
+=== contains
+
+*Description*: [.description]#Returns `true` if the Subject contains the value of the argument anywhere in the value.#
+
+*Subject Type*: [.subject]#String#
+
+*Arguments*:
+
+ - [.argName]#_value_# : [.argDesc]#The value to search for#
+
+*Return Type*: [.returnType]#Boolean#
+
+*Examples*: If the "filename" attribute has the value "a brand new filename.txt", then the Expression
+ `${filename:contains('new')}` will return `true`. `${filename:contains('NEW')}` will
+ return `false`. `${filename:toUpper():contains('NEW')}` returns `true`.
+
+
+
+
+
+[.function]
+=== find
+
+*Description*: [.description]#Returns `true` if the Subject contains any sequence of characters that matches the
+ Regular Expression provided by the argument.#
+
+*Subject Type*: [.subject]#String#
+
+*Arguments*:
+
+ - [.argName]#_Regex_# : [.argDesc]#The Regular Expression (in the Java Pattern syntax) to match against the Subject#
+
+*Return Type*: [.returnType]#Boolean#
+
+*Examples*:
+
+If the "filename" attribute has the value "a brand new filename.txt", then the following
+Expressions will provide the following results:
+
+
+.find Examples
+|=======================================================================================
+| Expression | Value
+| `${filename:find('a [Bb]rand [Nn]ew')}` | `true`
+| `${filename:find('Brand.*')}` | `false`
+| `${filename:find('brand')}` | `true`
+|=======================================================================================
+
+
+
+
+
+[.function]
+=== matches
+
+*Description*: [.description]#Returns `true` if the Subject exactly matches the Regular Expression provided by the argument.#
+
+*Subject Type*: [.subject]#String#
+
+*Arguments*:
+
+ - [.argName]#_Regex_# : [.argDesc]#The Regular Expression (in the Java Pattern syntax) to match against the Subject#
+
+*Return Type*: [.returnType]#Boolean#
+
+*Examples*:
+
+If the "filename" attribute has the value "a brand new filename.txt", then the following
+Expressions will provide the following results:
+
+
+.matches Examples
+|=======================================================================================
+| Expression | Value
+| `${filename:matches('a.*txt')}` | `true`
+| `${filename:matches('brand')}` | `false`
+| `${filename:matches('.+brand.+')}` | `true`
+|=======================================================================================
+
+
+
+
+[.function]
+=== indexOf
+
+*Description*: [.description]#Returns the index of the first character in the Subject that matches the String value provided
+ as an argument. If the argument is found multiple times within the Subject, the value returned is the
+ starting index of the *first* occurrence.
+ If the argument cannot be found in the Subject, returns `-1`. The index is zero-based. This means that if
+ the search string is found at the beginning of the Subject, the value returned will be `0`, not `1`.#
+
+*Subject Type*: [.subject]#String#
+
+*Arguments*:
+
+ - [.argName]#_value_# : [.argDesc]#The value to search for in the Subject#
+
+*Return Type*: [.returnType]#Number#
+
+*Examples*: If the "filename" attribute has the value "a brand new filename.txt", then the following
+Expressions will provide the following results:
+
+
+
+.indexOf Examples
+|===============================================
+| Expression | Value
+| `${filename:indexOf('a.*txt')}` | `-1`
+| `${filename:indexOf('.')}` | `20`
+| `${filename:indexOf('a')}` | `0`
+| `${filename:indexOf(' ')}` | `1`
+|===============================================
+
+
+
+
+[.function]
+=== lastIndexOf
+
+*Description*: [.description]#Returns the index of the first character in the Subject that matches the String value provided
+ as an argument. If the argument is found multiple times within the Subject, the value returned is the
+ starting index of the *last* occurrence.
+ If the argument cannot be found in the Subject, returns `-1`. The index is zero-based. This means that if
+ the search string is found at the beginning of the Subject, the value returned will be `0`, not `1`.#
+
+*Subject Type*: [.subject]#String#
+
+*Arguments*:
+
+ - [.argName]#_value_# : [.argDesc]#The value to search for in the Subject#
+
+*Return Type*: [.returnType]#Number#
+
+*Examples*: If the "filename" attribute has the value "a brand new filename.txt", then the following
+Expressions will provide the following results:
+
+.lastIndexOf Examples
+|=======================================================================================
+| Expression | Value
+| `${filename:lastIndexOf('a.*txt')}` | `-1`
+| `${filename:lastIndexOf('.')}` | `20`
+| `${filename:lastIndexOf('a')}` | `17`
+| `${filename:lastIndexOf(' ')}` | `11`
+|=======================================================================================
+
+
+
+
+[[numbers]]
+== Mathematical Operations and Numeric Manipulation
+
+
+[.function]
+=== plus
+
+*Description*: [.description]#Adds a numeric value to the Subject. If either the argument or the Subject cannot be
+ coerced into a Number, returns `null`.#
+
+*Subject Type*: [.subject]#Number#
+
+*Arguments*:
+
+ - [.argName]#_Operand_# : [.argDesc]#The value to add to the Subject#
+
+*Return Type*: [.returnType]#Number#
+
+*Examples*: If the "fileSize" attribute has a value of 100, then the Expression `${fileSize:plus(1000)}`
+ will return the value `1100`.
+
+
+
+
+
+[.function]
+=== minus
+
+*Description*: [.description]#Subtracts a numeric value from the Subject.#
+
+*Subject Type*: [.subject]#Number#
+
+*Arguments*:
+
+ - [.argName]#_Operand_# : [.argDesc]#The value to subtract from the Subject#
+
+*Return Type*: [.returnType]#Number#
+
+*Examples*: If the "fileSize" attribute has a value of 100, then the Expression `${fileSize:minus(100)}`
+ will return the value `0`.
+
+
+
+
+
+[.function]
+=== multiply
+
+*Description*: [.description]#Multiplies a numeric value by the Subject and returns the product.#
+
+*Subject Type*: [.subject]#Number#
+
+*Arguments*:
+
+ - [.argName]#_Operand_# : [.argDesc]#The value to multiple the Subject by#
+
+*Return Type*: [.returnType]#Number#
+
+*Examples*: If the "fileSize" attribute has a value of 100, then the Expression `${fileSize:multiply(1024)}`
+ will return the value `102400`.
+
+
+
+
+[.function]
+=== divide
+
+*Description*: [.description]#Divides a numeric value by the Subject and returns the result, rounded down to the nearest integer.#
+
+*Subject Type*: [.subject]#Number#
+
+*Arguments*:
+
+ - [.argName]#_Operand_# : [.argDesc]#The value to add divide the Subject by#
+
+*Return Type*: [.returnType]#Number#
+
+*Examples*: If the "fileSize" attribute has a value of 100, then the Expression `${fileSize:divide(12)}`
+ will return the value `8`.
+
+
+
+
+[.function]
+=== mod
+
+*Description*: [.description]#Performs a modular division of the Subject by the argument. That is, this function will divide
+ the Subject by the value of the argument and return not the quotient but rather the remainder.#
+
+*Subject Type*: [.subject]#Number#
+
+*Arguments*:
+
+ - [.argName]#_Operand_# : [.argDesc]#The value to divide the Subject by#
+
+*Return Type*: [.returnType]#Number#
+
+*Examples*: If the "fileSize" attribute has a value of 100, then the Expression `${fileSize:mod(12)}`
+ will return the value `4`.
+
+
+
+
+
+[.function]
+=== toRadix
+
+*Description*: [.description]#Converts the Subject from a Base 10 number to a different Radix (or number base). An optional
+ second argument can be used to indicate the minimum number of characters to be used. If the converted value
+ has fewer than this number of characters, the number will be padded with leading zeroes.#
+
+*Subject Type*: [.subject]#Number#
+
+*Arguments*:
+
+ - [.argName]#_Desired Base_# : [.argDesc]#A Number between 2 and 36 (inclusive)#
+ - [.argName]#_Padding_# : [.argDesc]#Optional argument that specifies the minimum number of characters in the converted output#
+
+*Return Type*: [.returnType]#String#
+
+*Examples*: If the "fileSize" attributes has a value of 1024, then the following Expressions will yield
+ the following results:
+
+
+.toRadix Examples
+|=======================================================================================
+| Expression | Value
+| `${fileSize:toRadix(10)}` | `1024`
+| `${fileSize:toRadix(10, 1)}` | `1024`
+| `${fileSize:toRadix(10, 8)}` | `00001024`
+| `${fileSize:toRadix(16)}` | `400`
+| `${fileSize:toRadix(16, 8)}` | `00000400`
+| `${fileSize:toRadix(2)}` | `10000000000`
+| `${fileSize:toRadix(2, 16)}` | `0000010000000000`
+|=======================================================================================
+
+
+
+
+[[dates]]
+== Date Manipulation
+
+
+
+[[format]]
+[.function]
+=== format
+
+*Description*: [.description]#Formats a number as a date/time according to the format specified by the argument. The argument
+ must be a String that is a valid Java SimpleDateFormat format. The Subject is expected to be a Number that
+ represents the number of milliseconds since Midnight GMT January 1, 1970.#
+
+*Subject Type*: [.subject]#Number#
+
+*Arguments*:
+
+ - [.argName]#_format_# : [.argDesc]#The format to use in the Java SimpleDateFormat syntax#
+
+*Return Type*: [.returnType]#String#
+
+*Examples*: If the attribute "time" has the value "1420058163264", then the following Expressions will yield
+ the following results:
+
+.format Examples
+|============================================================================
+| Expression | Value
+| `${time:format("yyyy/MM/dd HH:mm:ss.SSS'Z'")}` | `2014/12/31 15:36:03.264Z`
+| `${time:format("yyyy/MM/dd")}` | `2014/12/31`
+| `${time:format("HH:mm:ss.SSS'Z'")}` | `15:36:03.264Z`
+| `${time:format("2014")}` | `2014`
+|============================================================================
+
+
+
+
+
+[.function]
+=== toDate
+
+*Description*: [.description]#Converts a String into a Number, based on the format specified by the argument. The argument
+ must be a String that is a valid Java SimpleDateFormat syntax. The Subject is expected to be a String
+ that is formatted according the argument. The return value is the numbr of milliseconds since
+ Midnight GMT January 1, 1979.#
+
+*Subject Type*: [.subject]#String#
+
+*Arguments*:
+
+ - [.argName]#_format_# : [.argDesc]#The current format to use when parsing the Subject, in the Java SimpleDateFormat syntax.#
+
+
+*Return Type*: [.returnType]#Number#
+
+*Examples*: If the attribute "year" has the value "2014" and the attribute "time" has the value "2014/12/31 15:36:03.264Z",
+ then the Expression `${year:toDate('yyyy')}` will return the number of milliseconds between Midnight GMT on January 1, 1970
+ and Midnight GMT on January 1, 2014. The Expression `${time:toDate("yyyy/MM/dd HH:mm:ss.SSS'Z'")}` will result in the
+ number of milliseconds between Midnight GMT on January 1, 1970 and 15:36:03.264 GMT on December 31, 2014.
+
+Often, this function is used in conjunction with the <<format>> function to change the format of a date/time. For example,
+if the attribute "date" has the value "12-24-2014" and we want to change the format to "2014/12/24", we can do so by
+chaining together the two functions: `${date:toDate('MM-dd-yyyy'):format('yyyy/MM/dd')}`.
+
+
+
+
+[.function]
+=== now
+
+*Description*: [.description]#The `now` function returns the current date and time as the number of milliseconds since Midnight GMT on
+ January 1, 1970.#
+
+*Subject Type*: [.subject]#No Subject#
+
+*Arguments*: No arguments
+
+*Return Type*: [.returnType]#Number#
+
+*Examples*: We can format the current date and time by using the `now` function in conjunction with the <<format>> function:
+ `${now():format('yyyy/MM/dd HH:mm:ss')}`.
+
+
+
+
+
+[[type_cast]]
+== Type Coercion
+
+[.function]
+=== toString
+
+*Description*: [.description]#Coerces the Subject into a String#
+
+*Subject Type*: [.subject]#Any type#
+
+*Arguments*: No arguments
+
+*Return Type*: [.returnType]#String#
+
+*Examples*: The Expression `${fileSize:toNumber():toString()}` converts the value of "fileSize" attribute to a number and
+ back to a String.
+
+
+
+
+
+[.function]
+=== toNumber
+
+*Description*: [.description]#Coerces the Subject into a Number#
+
+*Subject Type*: [.subject]#String#
+
+*Arguments*: No arguments
+
+*Return Type*: [.returnType]#Number#
+
+*Examples*: The Expression `${fileSize:toNumber()}` converts the String attribute value of "fileSize" to a number.
+
+
+
+
+
+
+[[subjectless]]
+== Subjectless Functions
+
+While the majority of functions in the Expression Language are called by using the syntax
+`${attributeName:function()}`, there exist a few functions that are not expected to have subjects.
+In this case, the attribute name is not present. For example, the IP address of the machine can
+be obtained by using the Expression `${ip()}`. All of the functions in this section are to be called
+without a subject. Attempting to call a subjectless function and provide it a subject will result in
+an error when validating the function.
+
+
+[.function]
+=== ip
+
+*Description*: [.description]#Returns the IP address of the machine.#
+
+*Subject Type*: [.subjectless]#No subject#
+
+*Arguments*: No arguments
+
+*Return Type*: [.returnType]#String#
+
+*Examples*: The IP address of the machine can be obtained by using the Expression `${ip()}`.
+
+
+
+
+
+[.function]
+=== hostname
+
+*Description*: [.description]#Returns the Hostname of the machine. An optional argument of type Boolean can be provided
+ to specify whether or not the Fully Qualified Domain Name should be used. If `false`, or not specified,
+ the hostname will not be fully qualified. If the argument is `true` but the fully qualified hostname
+ cannot be resolved, the simple hostname will be returned.#
+
+*Subject Type*: [.subjectless]#No subject#
+
+*Arguments*:
+
+ - [.argName]#_Fully Qualified_# : [.argDesc]#Optional parameter that specifies whether or not the hostname should be
+ fully qualified. If not specified, defaults to false.#
+
+*Return Type*: [.returnType]#String#
+
+*Examples*: The fully qualified hostname of the machine can be obtained by using the Expression `${hostname(true)}`,
+ while the simple hostname can be obtained by using either `${hostname(false)}` or simply `${hostname()}`.
+
+
+
+
+
+[.function]
+=== UUID
+
+*Description*: [.description]#Returns a randomly generated UUID.#
+
+*Subject Type*: [.subjectless]#No Subject#
+
+*Arguments*: No arguments
+
+*Return Type*: [.returnType]#String#
+
+*Examples*: ${UUID()} returns a value similar to de305d54-75b4-431b-adb2-eb6b9e546013
+
+
+
+
+
+[.function]
+=== nextInt
+
+*Description*: [.description]#Returns a one-up value (starting at 0) and increasing over the lifetime of the running instance of NiFi.
+ This value is not persisted across restarts and is not guaranteed to be unique across a cluster.
+ This value is considered "one-up" in that if called multiple times across the NiFi instance, the values will be sequential.
+ However, this counter is shared across all NiFi components, so calling this function multiple times from one Processor will
+ not guarantee sequential values within the context of a particular Processor.#
+
+*Subject Type*: [.subjectless]#No Subject#
+
+*Arguments*: No arguments
+
+*Return Type*: [.returnType]#Number#
+
+*Examples*: If the previous value returned by `nextInt` was `5`, the Expression `${nextInt():divide(2)}` obtains the next available
+ integer (6) and divides the result by 2, returning a value of `3`.
+
+
+
+[.function]
+=== literal
+
+*Description*: [.description]#Returns its argument as a literal String value. This is useful in order to treat a string or a number
+ at the beginning of an Expression as an actual value, rather than treating it as an attribute name. Additionally, it
+ can be used when the argument is an embedded Expression that we would then like to evaluate additional functions against.#
+
+*Subject Type*: [.subjectless]#No Subject#
+
+*Arguments*:
+
+ - [.argName]#_value_# : [.argDesc]#The value to be treated as a literal string, number, or boolean value.#
+
+*Return Type*: [.returnType]#String#
+
+*Examples*: `${literal(2):gt(1)}` returns `true`
+
+`${literal( ${allMatchingAttributes('a.*'):count()} ):gt(3)}` returns true if there are more than 3 attributes whose
+names begin with the letter `a`.
+
+
+[[multi]]
+== Evaluating Multiple Attributes
+
+When it becomes necessary to evaluate the same conditions against multiple attributes, this can be accomplished by means of the
+`and` and `or` functions. However, this quickly becomes tedious, error-prone, and difficult to maintain. For this reason, NiFi
+provides several functions for evaluating the same conditions against groups of attributes at the same time.
+
+
+
+
+[.function]
+=== anyAttribute
+
+*Description*: [.description]#Checks to see if any of the given attributes, match the given condition. This function has no subject and takes one or more
+ arguments that are the names of attributes to which the remainder of the Expression is to be applied. If any of the attributes specified,
+ when evaluated against the rest of the Expression, returns a value of `true`, then this function will return `true`. Otherwise, this function
+ will return `false`.#
+
+*Subject Type*: [.subjectless]#No Subject#
+
+*Arguments*:
+
+ - [.argName]#_Attribute Names_# : [.argDesc]#One or more attribute names to evaluate#
+
+
+*Return Type*: [.returnType]#Boolean#
+
+*Examples*: Given that the "abc" attribute contains the value "hello world", "xyz" contains "good bye world",
+ and "filename" contains "file.txt" consider the following examples:
+
+.anyAttribute Examples
+|=======================================================================
+| Expression | Value
+| `${anyAttribute("abc", "xyz"):contains("bye")}` | `true`
+| `${anyAttribute("filename","xyz"):toUpper():contains("e")}` | `false`
+|=======================================================================
+
+
+
+
+[.function]
+=== allAttributes
+
+*Description*: [.description]#Checks to see if any of the given attributes, match the given condition. This function has no subject and takes one or more
+ arguments that are the names of attributes to which the remainder of the Expression is to be applied. If all of the attributes specified,
+ when evaluated against the rest of the Expression, returns a value of `true`, then this function will return `true`. Otherwise, this function
+ will return `false`.#
+
+*Subject Type*: [.subjectless]#No Subject#
+
+*Arguments*:
+
+ - [.argName]#_Attribute Names_# : [.argDesc]#One or more attribute names to evaluate#
+
+*Return Type*: [.returnType]#Boolean#
+
+*Examples*: Given that the "abc" attribute contains the value "hello world", "xyz" contains "good bye world",
+ and "filename" contains "file.txt" consider the following examples:
+
+.allAttributes Example
+|=============================================================================
+| Expression | Value
+| `${allAttributes("abc", "xyz"):contains("world")}` | `true`
+| `${allAttributes("abc", "filename","xyz"):toUpper():contains("e")}` | `false`
+|=============================================================================
+
+
+
+
+
+[.function]
+=== anyMatchingAttribute
+
+*Description*: [.description]#Checks to see if any of the given attributes, match the given condition. This function has no subject and takes one or more
+ arguments that are Regular Expressions to match against attribute names. Any attribute whose name matches one of the supplied
+ Regular Expressions will be evaluated against the rest of the Expression. If any of the attributes specified,
+ when evaluated against the rest of the Expression, returns a value of `true`, then this function will return `true`. Otherwise, this function
+ will return `false`.#
+
+*Subject Type*: [.subjectless]#No Subject#
+
+*Arguments*:
+
+ - [.argName]#_Regex_# : [.argDesc]#One or more Regular Expressions (in the Java Pattern syntax) to evaluate against attribute names#
+
+
+*Return Type*: [.returnType]#Boolean#
+
+*Examples*: Given that the "abc" attribute contains the value "hello world", "xyz" contains "good bye world",
+ and "filename" contains "file.txt" consider the following examples:
+
+.anyMatchingAttribute Example
+|==============================================================
+| Expression | Value
+| `${anyMatchingAttribute("[ax].*"):contains('bye')}` | `true`
+| `${anyMatchingAttribute(".*"):isNull()}` | `false`
+|==============================================================
+
+
+
+
+
+[.function]
+=== allMatchingAttributes
+
+*Description*: [.description]#Checks to see if any of the given attributes, match the given condition. This function has no subject and takes one or more
+ arguments that are Regular Expressions to match against attribute names. Any attribute whose name matches one of the supplied
+ Regular Expressions will be evaluated against the rest of the Expression. If all of the attributes specified,
+ when evaluated against the rest of the Expression, return a value of `true`, then this function will return `true`. Otherwise, this function
+ will return `false`.#
+
+*Subject Type*: [.subjectless]#No Subject#
+
+ - [.argName]#_Regex_# : [.argDesc]#One or more Regular Expressions (in the Java Pattern syntax) to evaluate against attribute names#
+
+*Return Type*: [.returnType]#Boolean#
+
+*Examples*: Given that the "abc" attribute contains the value "hello world", "xyz" contains "good bye world",
+ and "filename" contains "file.txt" consider the following examples:
+
+.anyMatchingAttributes Examples
+|==============================================================
+| Expression | Value
+| `${allMatchingAttributes("[ax].*"):contains("world")}` | `true`
+| `${allMatchingAttributes(".*"):isNull()}` | `false`
+| `${allMatchingAttributes("f.*"):count()}` | `1`
+|==============================================================
+
+
+
+
+
+[.function]
+=== anyDelineatedValue
+
+*Description*: [.description]#Splits a String apart according to a delimiter that is provided, and then evaluates each of the values against
+ the rest of the Expression. If the Expression, when evaluated against any of the individual values, returns `true`, this
+ function returns `true`. Otherwise, the function returns `false`.#
+
+*Subject Type*: [.subjectless]#No Subject#
+
+*Arguments*:
+
+ - [.argName]#_Delineated Value_# : [.argDesc]#The value that is delineated. This is generally an embedded Expression,
+ though it does not have to be.#
+ - [.argName]#_Delimiter_# : [.argDesc]#The value to use to split apart the _delineatedValue_ argument.#
+
+*Return Type*: [.returnType]#Boolean#
+
+*Examples*: Given that the "number_list" attribute contains the value "1,2,3,4,5", and the "word_list" attribute contains the value "the,and,or,not",
+ consider the following examples:
+
+.anyDelineatedValue Examples
+|===============================================================================
+| Expression | Value
+| `${anyDelineatedValue("${number_list}", ","):contains("5")}` | `true`
+| `${anyDelineatedValue("this that and", ","):equals("${word_list}")}` | `false`
+|===============================================================================
+
+
+
+[.function]
+=== allDelineatedValues
+
+*Description*: [.description]#Splits a String apart according to a delimiter that is provided, and then evaluates each of the values against
+ the rest of the Expression. If the Expression, when evaluated against all of the individual values, returns `true` in each
+ case, then this function returns `true`. Otherwise, the function returns `false`.#
+
+*Subject Type*: [.subjectless]#No Subject#
+
+*Arguments*:
+
+ - [.argName]#_Delineated Value_# : [.argDesc]#The value that is delineated. This is generally
+ an embedded Expression, though it does not have to be.#
+
+ - [.argName]#_Delimiter_# : [.argDesc]#The value to use to split apart the _delineatedValue_ argument.#
+
+*Return Type*: [.returnType]#Boolean#
+
+*Examples*: Given that the "number_list" attribute contains the value "1,2,3,4,5", and the "word_list" attribute contains the value "those,known,or,not",
+ consider the following examples:
+
+.allDelineatedValues Examples
+|===============================================================================
+| Expression | Value
+| `${allDelineatedValues("${word_list}", ","):contains("o")}` | `true`
+| `${allDelineatedValues("${number_list}", ","):count()}` | `4`
+| `${allDelineatedValues("${number_list}", ","):matches("[0-9]+")}` | `true`
+| `${allDelineatedValues("${word_list}", ","):matches('e')}` | `false`
+|===============================================================================
+
+
+
+
+[.function]
+=== join
+
+*Description*: [.description]#Aggregate function that concatenates multiple values with the specified delimiter. This function
+ may be used only in conjunction with the `allAttributes`, `allMatchingAttributes`, and `allDelineatedValues`
+ functions.#
+
+*Subject Type*: [.subject]#String#
+
+*Arguments*:
+
+ - [.argName]#_Delimiter_# : [.argDesc]#The String delimiter to use when joining values#
+
+*Return Type*: [.returnType]#String#
+
+*Examples*: Given that the "abc" attribute contains the value "hello world", "xyz" contains "good bye world",
+ and "filename" contains "file.txt" consider the following examples:
+
+.join Examples
+|=======================================================================================
+| Expression | Value
+| `${allMatchingAttributes("[ax].*"):substringBefore(" "):join("-")}` | `hello-good`
+| `${allAttributes("abc", "xyz"):join(" now")}` | `hello world nowgood bye world now`
+|=======================================================================================
+
+
+
+
+
+
+[.function]
+=== count
+
+*Description*: [.description]#Aggregate function that counts the number of non-null, non-false values returned by the
+ `allAttributes`, `allMatchingAttributes`, and `allDelineatedValues`. This function
+ may be used only in conjunction with the `allAttributes`, `allMatchingAttributes`, and `allDelineatedValues`
+ functions.#
+
+*Subject Type*: [.subject]#Any#
+
+*Arguments*: No arguments
+
+*Return Type*: [.returnType]#Number#
+
+*Examples*: Given that the "abc" attribute contains the value "hello world", "xyz" contains "good bye world",
+ and "number_list" contains "1,2,3,4,5" consider the following examples:
+
+.count Examples
+|===========================================================================
+| Expression | Value
+| `${allMatchingAttributes("[ax].*"):substringBefore(" "):count()}` | `2`
+| `${allAttributes("abc", "xyz"):contains("world"):count()}` | `1`
+| `${allDelineatedValues(${number_list}, ","):count()}` | `5`
+| `${allAttributes("abc", "non-existent-attr", "xyz"):count()}` | `2`
+| `${allMatchingAttributes(".*"):length():gt(10):count()}` | `2`
+|===========================================================================
+
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/add-controller-service-window.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/add-controller-service-window.png b/nifi-docs/src/main/asciidoc/images/add-controller-service-window.png
new file mode 100644
index 0000000..b727a9e
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/add-controller-service-window.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/add-processor-with-tag-cloud.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/add-processor-with-tag-cloud.png b/nifi-docs/src/main/asciidoc/images/add-processor-with-tag-cloud.png
new file mode 100644
index 0000000..a0bbadb
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/add-processor-with-tag-cloud.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/add-processor.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/add-processor.png b/nifi-docs/src/main/asciidoc/images/add-processor.png
new file mode 100644
index 0000000..a043f38
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/add-processor.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/addConnect.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/addConnect.png b/nifi-docs/src/main/asciidoc/images/addConnect.png
new file mode 100644
index 0000000..b33a305
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/addConnect.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/comments-tab.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/comments-tab.png b/nifi-docs/src/main/asciidoc/images/comments-tab.png
new file mode 100644
index 0000000..b6ed34b
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/comments-tab.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/components.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/components.png b/nifi-docs/src/main/asciidoc/images/components.png
new file mode 100644
index 0000000..b3a9769
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/components.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/configure-controller-service-properties.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/configure-controller-service-properties.png b/nifi-docs/src/main/asciidoc/images/configure-controller-service-properties.png
new file mode 100644
index 0000000..c0b4d83
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/configure-controller-service-properties.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/configure-controller-service-settings.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/configure-controller-service-settings.png b/nifi-docs/src/main/asciidoc/images/configure-controller-service-settings.png
new file mode 100644
index 0000000..d97d5a1
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/configure-controller-service-settings.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/configure-reporting-task-properties.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/configure-reporting-task-properties.png b/nifi-docs/src/main/asciidoc/images/configure-reporting-task-properties.png
new file mode 100644
index 0000000..cb303a1
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/configure-reporting-task-properties.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/configure-reporting-task-settings.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/configure-reporting-task-settings.png b/nifi-docs/src/main/asciidoc/images/configure-reporting-task-settings.png
new file mode 100644
index 0000000..0f9474d
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/configure-reporting-task-settings.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/connection-settings.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/connection-settings.png b/nifi-docs/src/main/asciidoc/images/connection-settings.png
new file mode 100644
index 0000000..9aae93a
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/connection-settings.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/controller-services-edit-buttons.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/controller-services-edit-buttons.png b/nifi-docs/src/main/asciidoc/images/controller-services-edit-buttons.png
new file mode 100644
index 0000000..5ca61d9
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/controller-services-edit-buttons.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/controller-services-edit-buttons2.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/controller-services-edit-buttons2.png b/nifi-docs/src/main/asciidoc/images/controller-services-edit-buttons2.png
new file mode 100644
index 0000000..ee3a5f7
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/controller-services-edit-buttons2.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/controller-services-tab.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/controller-services-tab.png b/nifi-docs/src/main/asciidoc/images/controller-services-tab.png
new file mode 100644
index 0000000..f6d76b8
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/controller-services-tab.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/controller-settings-button.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/controller-settings-button.png b/nifi-docs/src/main/asciidoc/images/controller-settings-button.png
new file mode 100644
index 0000000..a4f7e92
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/controller-settings-button.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/create-connection.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/create-connection.png b/nifi-docs/src/main/asciidoc/images/create-connection.png
new file mode 100644
index 0000000..7ca40ae
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/create-connection.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/create-service-ssl-context.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/create-service-ssl-context.png b/nifi-docs/src/main/asciidoc/images/create-service-ssl-context.png
new file mode 100644
index 0000000..8db2ea6
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/create-service-ssl-context.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/edit-property-dropdown.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/edit-property-dropdown.png b/nifi-docs/src/main/asciidoc/images/edit-property-dropdown.png
new file mode 100644
index 0000000..017805b
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/edit-property-dropdown.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/edit-property-textarea.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/edit-property-textarea.png b/nifi-docs/src/main/asciidoc/images/edit-property-textarea.png
new file mode 100644
index 0000000..cc8668a
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/edit-property-textarea.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/event-attributes.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/event-attributes.png b/nifi-docs/src/main/asciidoc/images/event-attributes.png
new file mode 100644
index 0000000..2ac4f43
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/event-attributes.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/event-content.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/event-content.png b/nifi-docs/src/main/asciidoc/images/event-content.png
new file mode 100644
index 0000000..f7b32a5
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/event-content.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/event-details.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/event-details.png b/nifi-docs/src/main/asciidoc/images/event-details.png
new file mode 100644
index 0000000..066aa99
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/event-details.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/expand-event.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/expand-event.png b/nifi-docs/src/main/asciidoc/images/expand-event.png
new file mode 100644
index 0000000..95eb8ab
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/expand-event.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/expanded-events.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/expanded-events.png b/nifi-docs/src/main/asciidoc/images/expanded-events.png
new file mode 100644
index 0000000..18bec5d
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/expanded-events.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/find-parents.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/find-parents.png b/nifi-docs/src/main/asciidoc/images/find-parents.png
new file mode 100644
index 0000000..57fe72e
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/find-parents.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/iconAlert.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconAlert.png b/nifi-docs/src/main/asciidoc/images/iconAlert.png
new file mode 100644
index 0000000..5020c31
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconAlert.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/iconConnection.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconConnection.png b/nifi-docs/src/main/asciidoc/images/iconConnection.png
new file mode 100644
index 0000000..4b6180b
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconConnection.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/iconDelete.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconDelete.png b/nifi-docs/src/main/asciidoc/images/iconDelete.png
new file mode 100644
index 0000000..dc13227
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconDelete.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/iconDisable.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconDisable.png b/nifi-docs/src/main/asciidoc/images/iconDisable.png
new file mode 100644
index 0000000..17a4ee4
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconDisable.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/iconEdit.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconEdit.png b/nifi-docs/src/main/asciidoc/images/iconEdit.png
new file mode 100644
index 0000000..e191e72
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconEdit.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/iconEnable.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconEnable.png b/nifi-docs/src/main/asciidoc/images/iconEnable.png
new file mode 100644
index 0000000..58e3436
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconEnable.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/iconExport.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconExport.png b/nifi-docs/src/main/asciidoc/images/iconExport.png
new file mode 100644
index 0000000..4a03bec
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconExport.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/iconFlowHistory.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconFlowHistory.png b/nifi-docs/src/main/asciidoc/images/iconFlowHistory.png
new file mode 100644
index 0000000..4ac29c0
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconFlowHistory.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/iconFunnel.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconFunnel.png b/nifi-docs/src/main/asciidoc/images/iconFunnel.png
new file mode 100644
index 0000000..98f9b9d
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconFunnel.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/iconInfo.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconInfo.png b/nifi-docs/src/main/asciidoc/images/iconInfo.png
new file mode 100644
index 0000000..f192ff5
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconInfo.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/iconInputPort.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconInputPort.png b/nifi-docs/src/main/asciidoc/images/iconInputPort.png
new file mode 100644
index 0000000..d4efd97
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconInputPort.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/iconInputPortSmall.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconInputPortSmall.png b/nifi-docs/src/main/asciidoc/images/iconInputPortSmall.png
new file mode 100644
index 0000000..1bbc079
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconInputPortSmall.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/iconLabel.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconLabel.png b/nifi-docs/src/main/asciidoc/images/iconLabel.png
new file mode 100644
index 0000000..6ba3c13
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconLabel.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/iconLineage.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconLineage.png b/nifi-docs/src/main/asciidoc/images/iconLineage.png
new file mode 100644
index 0000000..cfbc75b
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconLineage.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/iconNewTemplate.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconNewTemplate.png b/nifi-docs/src/main/asciidoc/images/iconNewTemplate.png
new file mode 100644
index 0000000..0e6b74c
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconNewTemplate.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/iconNotSecure.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconNotSecure.png b/nifi-docs/src/main/asciidoc/images/iconNotSecure.png
new file mode 100644
index 0000000..77ae0a2
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconNotSecure.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/iconOutputPort.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconOutputPort.png b/nifi-docs/src/main/asciidoc/images/iconOutputPort.png
new file mode 100644
index 0000000..f3d8a66
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconOutputPort.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/iconOutputPortSmall.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconOutputPortSmall.png b/nifi-docs/src/main/asciidoc/images/iconOutputPortSmall.png
new file mode 100644
index 0000000..d9338b3
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconOutputPortSmall.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/iconProcessGroup.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconProcessGroup.png b/nifi-docs/src/main/asciidoc/images/iconProcessGroup.png
new file mode 100644
index 0000000..c587341
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconProcessGroup.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/iconProcessor.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconProcessor.png b/nifi-docs/src/main/asciidoc/images/iconProcessor.png
new file mode 100644
index 0000000..2ca062c
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconProcessor.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/iconProvenance.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconProvenance.png b/nifi-docs/src/main/asciidoc/images/iconProvenance.png
new file mode 100644
index 0000000..32052e9
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconProvenance.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/iconRemoteProcessGroup.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconRemoteProcessGroup.png b/nifi-docs/src/main/asciidoc/images/iconRemoteProcessGroup.png
new file mode 100644
index 0000000..5205470
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconRemoteProcessGroup.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/iconResize.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconResize.png b/nifi-docs/src/main/asciidoc/images/iconResize.png
new file mode 100644
index 0000000..d6fd0a3
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconResize.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/iconRun.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconRun.png b/nifi-docs/src/main/asciidoc/images/iconRun.png
new file mode 100644
index 0000000..02715e0
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconRun.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/iconSecure.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconSecure.png b/nifi-docs/src/main/asciidoc/images/iconSecure.png
new file mode 100644
index 0000000..3f165a8
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconSecure.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/iconSettings.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconSettings.png b/nifi-docs/src/main/asciidoc/images/iconSettings.png
new file mode 100644
index 0000000..8fdaffc
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconSettings.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/iconStop.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconStop.png b/nifi-docs/src/main/asciidoc/images/iconStop.png
new file mode 100644
index 0000000..4f8f9c7
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconStop.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/iconSummary.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconSummary.png b/nifi-docs/src/main/asciidoc/images/iconSummary.png
new file mode 100644
index 0000000..973933c
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconSummary.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/iconTemplate.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconTemplate.png b/nifi-docs/src/main/asciidoc/images/iconTemplate.png
new file mode 100644
index 0000000..66d2caa
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconTemplate.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/iconTransmissionActive.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconTransmissionActive.png b/nifi-docs/src/main/asciidoc/images/iconTransmissionActive.png
new file mode 100644
index 0000000..269cff8
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconTransmissionActive.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/iconTransmissionInactive.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconTransmissionInactive.png b/nifi-docs/src/main/asciidoc/images/iconTransmissionInactive.png
new file mode 100644
index 0000000..0e1fd7b
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconTransmissionInactive.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/iconUsers.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconUsers.png b/nifi-docs/src/main/asciidoc/images/iconUsers.png
new file mode 100644
index 0000000..fc99a12
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconUsers.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/iconViewDetails.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/iconViewDetails.png b/nifi-docs/src/main/asciidoc/images/iconViewDetails.png
new file mode 100644
index 0000000..5231e73
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/iconViewDetails.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/instantiate-template-description.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/instantiate-template-description.png b/nifi-docs/src/main/asciidoc/images/instantiate-template-description.png
new file mode 100644
index 0000000..a238148
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/instantiate-template-description.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/instantiate-template.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/instantiate-template.png b/nifi-docs/src/main/asciidoc/images/instantiate-template.png
new file mode 100644
index 0000000..435d562
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/instantiate-template.png differ
[19/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-commons/pom.xml b/nifi-commons/pom.xml
new file mode 100644
index 0000000..dc6dde7
--- /dev/null
+++ b/nifi-commons/pom.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-commons</artifactId>
+ <packaging>pom</packaging>
+ <modules>
+ <module>nifi-data-provenance-utils</module>
+ <module>nifi-flowfile-packager</module>
+ <module>nifi-expression-language</module>
+ <module>nifi-logging-utils</module>
+ <module>nifi-properties</module>
+ <module>nifi-security-utils</module>
+ <module>nifi-socket-utils</module>
+ <module>nifi-utils</module>
+ <module>nifi-web-utils</module>
+ <module>nifi-processor-utilities</module>
+ <module>nifi-write-ahead-log</module>
+ <module>nifi-site-to-site-client</module>
+ <module>nifi-hl7-query-language</module>
+ </modules>
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/LICENSE
----------------------------------------------------------------------
diff --git a/nifi-docs/LICENSE b/nifi-docs/LICENSE
new file mode 100644
index 0000000..f6dd49c
--- /dev/null
+++ b/nifi-docs/LICENSE
@@ -0,0 +1,235 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+APACHE NIFI SUBCOMPONENTS:
+
+The Apache NiFi project contains subcomponents with separate copyright
+notices and license terms. Your use of the source code for the these
+subcomponents is subject to the terms and conditions of the following
+licenses.
+
+This product bundles source from 'Asciidoctor'. Specifically the 'asciidoc-mod.css'.
+The source is available under an MIT LICENSE.
+
+ Copyright (C) 2012-2015 Dan Allen, Ryan Waldron and the Asciidoctor Project
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+
+
+
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/NOTICE
----------------------------------------------------------------------
diff --git a/nifi-docs/NOTICE b/nifi-docs/NOTICE
new file mode 100644
index 0000000..b263bd8
--- /dev/null
+++ b/nifi-docs/NOTICE
@@ -0,0 +1,5 @@
+nifi-docs
+Copyright 2014-2015 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-docs/pom.xml b/nifi-docs/pom.xml
new file mode 100644
index 0000000..42a1775
--- /dev/null
+++ b/nifi-docs/pom.xml
@@ -0,0 +1,152 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+ <packaging>pom</packaging>
+ <artifactId>nifi-docs</artifactId>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-resources-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-asciidoc</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <resources>
+ <resource>
+ <directory>src/main/asciidoc</directory>
+ </resource>
+ </resources>
+ <outputDirectory>${project.build.directory}/asciidoc</outputDirectory>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.asciidoctor</groupId>
+ <artifactId>asciidoctor-maven-plugin</artifactId>
+ <version>1.5.2</version>
+ <executions>
+ <execution>
+ <id>output-html</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>process-asciidoc</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <sourceDirectory>${project.build.directory}/asciidoc</sourceDirectory>
+ <backend>html5</backend>
+ <attributes>
+ <imagesdir>./images</imagesdir>
+ <icons>font</icons>
+ <toc>true</toc>
+ <docVersion>${project.version}</docVersion>
+ <sectanchors>true</sectanchors>
+ <idprefix />
+ <idseparator>-</idseparator>
+ <docinfo1>true</docinfo1>
+ <stylesheet>asciidoc-mod.css</stylesheet>
+ </attributes>
+ </configuration>
+ </plugin>
+ <!-- This plugin is used to insert the Apache License into the output HMTL because
+ AsciiDoc doesn't appear to provide a mechanism for doing this. -->
+ <plugin>
+ <groupId>com.google.code.maven-replacer-plugin</groupId>
+ <artifactId>replacer</artifactId>
+ <version>1.5.3</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>replace</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <filesToInclude>${project.build.directory}/generated-docs/**.html</filesToInclude>
+ <regex>true</regex>
+ <regexFlags>
+ <regexFlag>DOTALL</regexFlag>
+ <regexFlag>MULTILINE</regexFlag>
+ </regexFlags>
+ <token>^(.*)$</token>
+ <value>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ $1
+ </value>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes combine.children="append">
+ <!-- MIT license confirmed. Excluding due to parse error-->
+ <exclude>src/main/asciidoc/asciidoc-mod.css</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <attach>true</attach>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make shared resource</id>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <phase>package</phase>
+ <configuration>
+ <descriptors>
+ <descriptor>src/main/assembly/dependencies.xml</descriptor>
+ </descriptors>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/administration-guide.adoc
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/administration-guide.adoc b/nifi-docs/src/main/asciidoc/administration-guide.adoc
new file mode 100644
index 0000000..679493a
--- /dev/null
+++ b/nifi-docs/src/main/asciidoc/administration-guide.adoc
@@ -0,0 +1,692 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+NiFi System Administrator's Guide
+=================================
+Apache NiFi Team <de...@nifi.apache.org>
+:homepage: http://nifi.apache.org
+
+System Requirements
+-------------------
+Apache NiFi can run on something as simple as a laptop, but it can also be clustered across many enterprise-class servers. Therefore, the amount of hardware and memory needed will depend on the size and nature of the dataflow involved. The data is stored on disk while NiFi is processing it. So NiFi needs to have sufficient disk space allocated for its various repositories, particularly the content repository, flowfile repository, and provenance repository (see the <<system_properties>> section for more information about these repositories). NiFi has the following minimum system requirements:
+
+* Requires Java 7 or newer
+* Supported Operating Systems:
+** Linux
+** Unix
+** Windows
+** Mac OS X
+* Supported Web Browsers:
+** Internet Explorer 9+ (see note below)
+** Mozilla FireFox 24+
+** Google Chrome 36+
+** Safari 8
+
+Note that there is a known issue in Internet Explorer (IE) 10 and 11 that can cause problems when moving items on the NiFi graph. If you encounter this problem, we suggest using a browser other than IE. This known issue is described here: https://connect.microsoft.com/IE/Feedback/Details/1050422.
+
+How to install and start NiFi
+-----------------------------
+
+* Linux/Unix/OS X
+** Decompress and untar into desired installation directory
+** Make any desired edits in files found under <installdir>/conf
+*** At a minimum, we recommend editing the _nifi.properties_ file and entering a password for the nifi.sensitive.props.key (see <<system_properties>> below)
+** From the <installdir>/bin directory, execute the following commands by typing ./nifi.sh <command>:
+*** start: starts NiFi in the background
+*** stop: stops NiFi that is running in the background
+*** status: provides the current status of NiFi
+*** run: runs NiFi in the foreground and waits for a Ctrl-C to initiate shutdown of NiFi
+*** install: installs NiFi as a service that can then be controlled via
+**** service nifi start
+**** service nifi stop
+**** service nifi status
+
+* Windows
+** Decompress into the desired installation directory
+** Make any desired edits in the files found under <installdir>/conf
+*** At a minimum, we recommend editing the _nifi.properties_ file and entering a password for the nifi.sensitive.props.key (see <<system_properties>> below)
+** Navigate to the <installdir>/bin directory
+** Double-click run-nifi.bat. This runs NiFi in the foreground and waits for a Ctrl-C to initiate shutdown of NiFi
+** Alternatively, to start NiFi in the background, double-click start-nifi.bat
+** To stop NiFi running in the background, double-click stop-nifi.bat
+** To see the current status of NiFi, double-click status-nifi.bat
+
+
+When NiFi first starts up, the following files and directories are created:
+
+* content_repository
+* database_repository
+* flowfile_repository
+* provenance_repository
+* work directory
+* logs directory
+* Within the conf directory, the _flow.xml.gz_ file and the templates directory are created
+
+See the <<system_properties>> section of this guide for more information about configuring NiFi repositories and configuration files.
+
+
+Configuration Best Practices
+----------------------------
+NOTE: If you are running on Linux, consider these best practices. Typical Linux defaults are not necessarily well tuned for the needs of an IO intensive application like NiFi. For all of these areas, your distribution's requirements may vary. Use these sections as advice, but
+consult your distribution-specific documentation for how best to achieve these recommendations.
+
+Maximum File Handles::
+NiFi will at any one time potentially have a very large number of file handles open. Increase the limits by
+editing '/etc/security/limits.conf' to add
+something like
+----
+* hard nofile 50000
+* soft nofile 50000
+----
+Maximum Forked Processes::
+NiFi may be configured to generate a significant number of threads. To increase the allowable number edit '/etc/security/limits.conf'
+----
+* hard nproc 10000
+* soft nproc 10000
+----
+And your distribution may require an edit to /etc/security/limits.d/90-nproc.conf by adding
+----
+* soft nproc 10000
+----
+
+Increase the number of TCP socket ports available::
+This is particularly important if your flow will be setting up and tearing
+down a large number of sockets in small period of time.
+----
+sudo sysctl -w net.ipv4.ip_local_port_range="10000 65000"
+----
+
+Set how long sockets stay in a TIMED_WAIT state when closed::
+You don't want your sockets to sit and linger too long given that you want to be
+able to quickly setup and teardown new sockets. It is a good idea to read more about
+it but to adjust do something like
+----
+sudo sysctl -w net.ipv4.netfilter.ip_conntrack_tcp_timeout_time_wait="1"
+----
+
+Tell Linux you never want NiFi to swap::
+Swapping is fantastic for some applications. It isn't good for something like
+NiFi that always wants to be running. To tell Linux you'd like swapping off you
+can edit '/etc/sysctl.conf' to add the following line
+----
+vm.swappiness = 0
+----
+For the partitions handling the various NiFi repos turn off things like 'atime'.
+Doing so can cause a surprising bump in throughput. Edit the '/etc/fstab' file
+and for the partition(s) of interest add the 'noatime' option.
+
+
+Security Configuration
+----------------------
+
+NiFi provides several different configuration options for security purposes. The most important properties are those under the
+"security properties" heading in the _nifi.properties_ file. In order to run securely, the following properties must be set:
+
+[options="header,footer"]
+|==================================================================================================================================================
+| Property Name | Description
+|`nifi.security.keystore` | Filename of the Keystore that contains the server's private key.
+|`nifi.security.keystoreType` | The type of Keystore. Must be either `PKCS12` or `JKS`.
+|`nifi.security.keystorePasswd` | The password for the Keystore.
+|`nifi.security.keyPasswd` | The password for the certificate in the Keystore. If not set, the value of `nifi.security.keystorePasswd` will be used.
+|`nifi.security.truststore` | Filename of the Truststore that will be used to authorize those connecting to NiFi. If not set, all who
+ attempt to connect will be provided access as the 'Anonymous' user.
+|`nifi.security.truststoreType` | The type of the Truststore. Must be either `PKCS12` or `JKS`.
+|`nifi.security.truststorePasswd` | The password for the Truststore.
+|`nifi.security.needClientAuth` | Specifies whether or not connecting clients must authenticate themselves. If the Truststore properties are not set,
+ this must be `false`. Otherwise, a value of `true` indicates that users will be authenticated and must have
+ certificates that are trusted by the Truststore loaded into their web browsers. A value of `false` indicates
+ that all users should be given access as the 'Anonymous' user.
+|==================================================================================================================================================
+
+Once the above properties have been configured, we can enable the User Interface to be accessed over HTTPS instead of HTTP. This is accomplished
+by setting the `nifi.web.https.host` and `nifi.web.https.port` properties. The `nifi.web.https.host` property indicates which hostname the server
+should run on. This allows admins to configure the application to run only on specific network interfaces. If it is desired that the HTTPS interface
+be accessible from all network interfaces, a value of `0.0.0.0` should be used.
+
+NOTE: It is important when enabling HTTPS that the `nifi.web.http.port` property be unset.
+
+Now that the User Interface has been secured, we can easily secure Site-to-Site connections and inner-cluster communications, as well. This is
+accomplished by setting the `nifi.remote.input.secure` and `nifi.cluster.protocol.is.secure` properties, respectively, to `true`.
+
+
+Controlling Levels of Access
+----------------------------
+
+Once NiFi is configured to run securely as discussed in the previous section, it is necessary
+to configure who will have access to the system and what types of access those people will have.
+NiFi controls this through the user of an 'Authority Provider.' The Authority Provider is a pluggable
+mechanism for providing authorizations to different users. Which Authority Provider to use is configured
+using two properties in the _nifi.properties_ file.
+
+The `nifi.authority.provider.configuration.file` property specifies the configuration file for Authority Providers.
+The `nifi.security.user.authority.provider` property indicates which of the configured Authority Providers should be
+used.
+
+By default, the `file-provider` Authority Provider is selected and is configured to use the permissions granted in
+the _authorized-users.xml_ file. This is typically sufficient for instances of NiFi that are run in "standalone" mode.
+If the NiFi instance is configured to run in a cluster, the node will typically use the `cluster-node-provider`
+Provider and the Cluster Manager will typically use the `cluster-ncm-provider` Provider. Both of these Providers
+have a default configuration in the _authority-providers.xml_ file but are commented out.
+
+When using the `cluster-node-provider` Provider, all of the authorization is provided by the Cluster Manager. In this
+way, the configuration only has to be maintained in one place and will be consistent across the entire cluster.
+
+When configuring the Cluster Manager or a standalone node, it is necessary to manually designate an ADMIN user
+in the _authorized-users.xml_ file, which is located in the root installation's conf directory.
+After this ADMIN user has been added, s/he may grant access
+to other users, systems, and other instances of NiFi, through the User Interface (UI) without having to manually edit the _authorized-users.xml_
+file. If you are the administrator, you would add yourself as the ADMIN user in this file.
+
+Open the _authorized-users.xml_ file in a text editor. You will notice that it includes a template
+to guide you, with example entries that are commented out.
+
+It is only necessary to manually add one user, the ADMIN user, to this file.
+So, at a minimum, the following example entry should be included and contain the user Distinguished Name (DN)
+in place of "user dn - read only and admin":
+
+----
+<users>
+ <user dn="[user dn - read only and admin]">
+ <role name="ROLE_ADMIN"/>
+ </user>
+</users>
+----
+
+Here is an example entry using the name John Smith:
+
+----
+<users>
+ <user dn="cn=John Smith,ou=people,dc=example,dc=com">
+ <role name="ROLE_ADMIN"/>
+ </user>
+</users>
+----
+
+After the _authorized-users.xml_ file has been edited and saved, restart NiFi.
+Once the application starts, the ADMIN user is
+able to access the UI at the HTTPS URL that is configured in the _nifi.properties_ file.
+
+From the UI, click on the Users icon ( image:iconUsers.png["Users", width=32] ) in the
+Management Toolbar (upper-right corner of the UI), and the User Management Page opens.
+
+The ADMIN user should be listed. Click on the pencil icon to see this user's role(s). You may edit the
+roles by selecting the appropriate checkboxes.
+
+The following roles are available in NiFi:
+
+[options="header,footer"]
+|========================================================================================================
+| Role Name | Description
+| Administrator | Administrator is able to configure thread pool sizes and user accounts as well as
+ purge the dataflow change history.
+| Data Flow Manager | Data Flow Manager is given the ability to manipulate the dataflow. S/he is able to
+ add, remove, and manipulate components on the graph; add, remove, and manipulate
+ Controller Services and Reporting Tasks; create and manage templates;
+ view statistics; and view the bulletin board.
+| Read Only | Users with Read Only access are able to view the dataflow but are unable to change anything.
+| Provenance | Users with Provenance access are able to query the Data Provenance repository and view
+ the lineage of data. Additionally, this role provides the ability to view or download
+ the content of a FlowFile from a Provenance event (assuming that the content is still
+ available in the Content Repository and that the Authority Provider also grants access).
+ This access is not provided to users with Read Only
+ (unless the user has both Read Only and Provenance roles) because the information provided
+ to users with this role can potentially be very sensitive in nature, as all FlowFile attributes
+ and data are exposed. In order to Replay a Provenance event, a user is required to have both
+ the Provenance role as well as the Data Flow Manager role.
+| NiFi | The NiFi Role is intended to be assigned to machines that will interact with an instance of NiFi
+ via Site-to-Site. This role provides the ability to send data to or retrieve data from Root
+ Group Ports (but only those that they are given permissions to interact with - see the User Guide
+ for more information on providing access to specific Ports) as well as obtain information about
+ which Ports exist. Note that this role allows the client to know only about the Ports that it
+ has permissions to interact with.
+| Proxy | The Proxy Role is assigned to a system in order to grant that system permission to make requests
+ on behalf of a user. For instance, if an HTTP proxy service is used to gain access to the system,
+ the certificate being used by that service can be given the Proxy Role.
+|========================================================================================================
+
+
+When users want access to the NiFi UI, they navigate to the configured URL and are
+prompted to request access. When someone has requested access, the ADMIN user sees a star
+on the Users icon in the Management Toolbar, alerting the ADMIN to the fact that a request is
+pending. Upon opening the User Management Page, the pending request is visible, and the ADMIN
+can grant access and click on the pencil icon to set the user's roles appropriately.
+
+The ADMIN may also select multiple users and add them to a "Group". Hold down the Shift key and select
+multiple users, then click the `Group` button in the upper-right corner of the User Management Page.
+Then, provide a name for the group.
+
+The group feature is especially useful when a remote NiFi cluster is connecting to this NiFi using
+a Remote Process Group. In that scenario, all the nodes
+in the remote cluster can be included in the same group. When the ADMIN wants to grant port access to the remote
+cluster, s/he can grant it to the group and avoid having to grant it individually to each node in the cluster.
+
+
+[[clustering]]
+Clustering Configuration
+------------------------
+
+This section provides a quick overview of NiFi Clustering and instructions on how to set up a basic cluster. In the future, we hope to provide supplemental documentation that covers the NiFi Cluster Architecture in depth.
+
+The design of NiFi clustering is a simple master/slave model where there is a master and one or more slaves.
+While the model is that of master and slave, if the master dies, the slaves are all instructed to continue operating
+as they were to ensure the dataflow remains live. The absence of the master simply means new slaves cannot join the
+cluster and cluster flow changes cannot occur until the master is restored. In NiFi clustering, we call the master
+the NiFi Cluster Manager (NCM), and the slaves are called Nodes. See a full description of each in the Terminology section below.
+
+*Why Cluster?* +
+
+NiFi Administrators or Dataflow Managers (DFMs) may find that using one instance of NiFi on a single server is not enough to process the amount of data they have. So, one solution is to run the same dataflow on multiple NiFi servers. However, this creates a management problem, because each time DFMs want to change or update the dataflow, they must make those changes on each server and then monitor each server individually. By clustering the NiFi servers, it's possible to have that increased processing capability along with a single interface through which to make dataflow changes and monitor the dataflow. Clustering allows the DFM to make each change only once, and that change is then replicated to all the nodes of the cluster. Through the single interface, the DFM may also monitor the health and status of all the nodes.
+
+NiFi Clustering is unique and has its own terminology. It's important to understand the following terms before setting up a cluster.
+
+[template="glossary", id="terminology"]
+*Terminology* +
+
+*NiFi Cluster Manager*: A NiFi Cluster Manager (NCM) is an instance of NiFi that provides the sole management point for the cluster. It communicates dataflow changes to the nodes and receives health and status information from the nodes. It also ensures that a uniform dataflow is maintained across the cluster. When DFMs manage a dataflow in a cluster, they do so through the User Interface of the NCM (i.e., via the URL of the NCM's User Interface). Fundamentally, the NCM keeps the state of the cluster consistent.
+
+*Nodes*: Each cluster is made up of the NCM and one or more nodes. The nodes do the actual data processing. (The NCM does not process any data; all data runs through the nodes.) While nodes are connected to a cluster, the DFM may not access the User Interface for any of the individual nodes. The User Interface of a node may only be accessed if the node is manually removed from the cluster.
+
+*Primary Node*: Every cluster has one Primary Node. On this node, it is possible to run "Isolated Processors" (see below). By default, the NCM will elect the first node that connects to the cluster as the Primary Node; however, the DFM may select a new node as the Primary Node in the Cluster Management page of the User Interface if desired. If the cluster restarts, the NCM will "remember" which node was the Primary Node and wait for that node to re-connect before allowing the DFM to make any changes to the dataflow. The ADMIN may adjust how long the NCM waits for the Primary Node to reconnect by adjusting the property _nifi.cluster.manager.safemode.duration_ in the _nifi.properties_ file, which is discussed in the <<system_properties>> section of this document.
+
+*Isolated Processors*: In a NiFi cluster, the same dataflow runs on all the nodes. As a result, every component in the flow runs on every node. However, there may be cases when the DFM would not want every processor to run on every node. The most common case is when using a processor that communicates with an external service using a protocol that does not scale well. For example, the GetSFTP processor pulls from a remote directory, and if the GetSFTP on every node in the cluster tries simultaneously to pull from the same remote directory, there could be race conditions. Therefore, the DFM could configure the GetSFTP on the Primary Node to run in isolation, meaning that it only runs on that node. It could pull in data and -with the proper dataflow configuration- load-balance it across the rest of the nodes in the cluster. Note that while this feature exists, it is also very common to simply use a standalone NiFi instance to pull data and feed it to the cluster. It just depends on th
e resources available and how the Administrator decides to configure the cluster.
+
+*Heartbeats*: The nodes communicate their health and status to the NCM via "heartbeats", which let the NCM know they are still connected to the cluster and working properly. By default, the nodes emit heartbeats to the NCM every 5 seconds, and if the NCM does not receive a heartbeat from a node within 45 seconds, it disconnects the node due to "lack of heartbeat". (The 5-second and 45-second settings are configurable in the _nifi.properties_ file. See the <<system_properties>> section of this document for more information.) The reason that the NCM disconnects the node is because the NCM needs to ensure that every node in the cluster is in sync, and if a node is not heard from regularly, the NCM cannot be sure it is still in sync with the rest of the cluster. If, after 45 seconds, the node does send a new heartbeat, the NCM will automatically reconnect the node to the cluster. Both the disconnection due to lack of heartbeat and the reconnection once a heartbeat is received are report
ed to the DFM in the NCM's User Interface.
+
+*Communication within the Cluster* +
+
+As noted, the nodes communicate with the NCM via heartbeats. The communication that allows the nodes to find the NCM may be set up as multicast or unicast; this is configured in the _nifi.properties_ file (See <<system_properties>> ). By default, unicast is used. It is important to note that the nodes in a NiFi cluster are not aware of each other. They only communicate with the NCM. Therefore, if one of the nodes goes down, the other nodes in the cluster will not automatically pick up the load of the missing node. It is possible for the DFM to configure the dataflow for failover contingencies; however, this is dependent on the dataflow design and does not happen automatically.
+
+When the DFM makes changes to the dataflow, the NCM communicates those changes to the nodes and waits for each node to respond, indicating that it has made the change on its local flow. If the DFM wants to make another change, the NCM will only allow this to happen once all the nodes have acknowledged that they've implemented the last change. This is a safeguard to ensure that all the nodes in the cluster have the correct and up-to-date flow.
+
+*Dealing with Disconnected Nodes* +
+
+A DFM may manually disconnect a node from the cluster. But if a node becomes disconnected for any other reason (such as due to lack of heartbeat), the NCM will show a bulletin on the User Interface, and the DFM will not be able to make any changes to the dataflow until the issue of the disconnected node is resolved. The DFM or the Administrator will need to troubleshoot the issue with the node and resolve it before any new changes may be made to the dataflow. However, it is worth noting that just because a node is disconnected does not mean that it is not working; it just means that the NCM cannot communicate with the node.
+
+
+*Basic Cluster Setup* +
+
+This section describes the setup for a simple two-node, non-secure, unicast cluster comprised of three instances of NiFi:
+
+* The NCM
+* Node 1
+* Node 2
+
+Administrators may install each instance on a separate server; however, it is also perfectly fine to install the NCM and one of the nodes on the same server, as the NCM is very lightweight. Just keep in mind that the ports assigned to each instance must not collide if the NCM and one of the nodes share the same server.
+
+For each instance, certain properties in the _nifi.properties_ file will need to be updated. In particular, the Web and Clustering properties should be evaluated for your situation and adjusted accordingly. All the properties are described in the <<system_properties>> section of this guide; however, in this section, we will focus on the minimum properties that must be set for a simple cluster.
+
+For all three instances, the Cluster Common Properties can be left with the default settings. Note, however, that if you change these settings, they must be set the same on every instance in the cluster (NCM and nodes).
+
+For the NCM, the minimum properties to configure are as follows:
+
+* Under the Web Properties, set either the http or https port that you want the NCM to run on. If the NCM and one of the nodes are on the same server, make sure this port is different from the web port used by the node.
+* Under the Cluster Manager Properties, set the following:
+** nifi.cluster.is.manager - Set this to _true_.
+** nifi.cluster.protocol.manager.port - Set this to an open port that is higher than 1024 (anything lower requires root). Take note of this setting, as you will need to reference it when you set up the nodes.
+
+For Node 1, the minimum properties to configure are as follows:
+
+* Under the Web Properties, set either the http or https port that you want Node 1 to run on. If the NCM is running on the same server, choose a different web port for Node 1. Also, consider whether you need to set the http or https host property.
+* Under Cluster Node Properties, set the following:
+** nifi.cluster.is.node - Set this to _true_.
+** nifi.cluster.node.address - Set this to the fully qualified hostname of the node. If left blank, it defaults to "localhost".
+** nifi.cluster.node.protocol.port - Set this to an open port that is higher than 1024 (anything lower requires root). If Node 1 and the NCM are on the same server, make sure this port is different from the nifi.cluster.protocol.manager.port.
+** nifi.cluster.node.unicast.manager.address - Set this to the NCM's fully qualified hostname.
+** nifi.cluster.node.unicast.manager.protocol.port - Set this to exactly the same port that was set on the NCM for the property nifi.cluster.manager.protocol.port.
+
+For Node 2, the minimum properties to configure are as follows:
+
+* Under the Web Properties, set either the http or https port that you want Node 2 to run on. Also, consider whether you need to set the http or https host property.
+* Under the Cluster Node Properties, set the following:
+** nifi.cluster.is.node - Set this to _true_.
+** nifi.cluster.node.address - Set this to the fully qualified hostname of the node. If left blank, it defaults to "localhost".
+** nifi.cluster.node.protocol.port - Set this to an open port that is higher than 1024 (anything lower requires root).
+** nifi.cluster.node.unicast.manager.address - Set this to the NCM's fully qualified hostname.
+** nifi.cluster.node.unicast.manager.protocol.port - Set this to exactly the same port that was set on the NCM for the property nifi.cluster.manager.protocol.port.
+
+Now, it is possible to start up the cluster. Technically, it does not matter which instance starts up first. However, you could start the NCM first, then Node 1 and then Node 2. Since the first node that connects is automatically elected as the Primary Node, this sequence should create a cluster where Node 1 is the Primary Node. Navigate to the URL for the NCM in your web browser, and the User Interface should look similar to the following:
+
+image:ncm.png["NCM User Interface", width=940]
+
+*Troubleshooting*
+
+If you encounter issues and your cluster does not work as described, investigate the nifi.app log and nifi.user log on both the NCM and the nodes. If needed, you can change the logging level to DEBUG by editing the conf/logback.xml file. Specifically, set the level="DEBUG" in the following line (instead of "INFO"):
+
+----
+ <logger name="org.apache.nifi.web.api.config" level="INFO"
+additivity="false">
+ <appender-ref ref="USER_FILE"/>
+ </logger>
+----
+
+
+
+[[system_properties]]
+System Properties
+-----------------
+The _nifi.properties_ file in the conf directory is the main configuration file for controlling how NiFi runs. This section provides an overview of the properties in this file and includes some notes on how to configure it in a way that will make upgrading easier. *After making changes to this file, restart NiFi in order
+for the changes to take effect.*
+
+NOTE: The contents of this file are relatively stable but do change from time to time. It is always a good idea to
+review this file when upgrading and pay attention for any changes. Consider configuring items
+below marked with an asterisk (*) in such a way that upgrading will be easier. For details, see a full discussion on upgrading
+at the end of this section. Note that values for periods of time and data sizes must include the unit of measure,
+for example "10 sec" or "10 MB", not simply "10".
+
+*Core Properties* +
+
+The first section of the _nifi.properties_ file is for the Core Properties. These properties apply to the core framework as a whole.
+
+|====
+|*Property*|*Description*
+|nifi.version|The version number of the current release. If upgrading but reusing this file, be sure to update this value.
+|nifi.flow.configuration.file*|The location of the flow configuration file (i.e., the file that contains what is currently displayed on the NiFi graph). The default value is ./conf/flow.xml.gz.
+|nifi.flow.configuration.archive.dir*|The location of the archive directory where backup copies of the flow.xml are saved. The default value is ./conf/archive.
+|nifi.flowcontroller.autoResumeState|Indicates whether -upon restart- the components on the NiFi graph should return to their last state. The default value is _true_.
+|nifi.flowcontroller.graceful.shutdown.period|Indicates the shutdown period. The default value is 10 sec.
+|nifi.flowservice.writedelay.interval|When many changes are made to the flow.xml, this property specifies how long to wait before writing out the changes, so as to batch the changes into a single write. The default value is 500 ms.
+|nifi.administrative.yield.duration|If a component allows an unexpected exception to escape, it is considered a bug. As a result, the framework will pause (or administratively yield) the component for this amount of time. This is done so that the component does not use up massive amounts of system resources, since it is known to have problems in the existing state. The default value is 30 sec.
+|nifi.authority.provider.configuration.file*|This is the location of the file that specifies how user access is authenticated. The default value is ./conf/authority-providers.xml.
+|nifi.reporting.task.configuration.file*|This is the location of the Reporting Tasks file. The default value is ./conf/reporting-tasks.xml.
+|nifi.controller.service.configuration.file*|This is the location of the Controller Services file. The default value is ./conf/controller-services.xml.
+|nifi.templates.directory*|This is the location of the directory where flow templates are saved. The default value is ./conf/templates.l
+|nifi.ui.banner.text|This is banner text that may be configured to display at the top of the User Interface. It is blank by default.
+|nifi.ui.autorefresh.interval|The interval at which the User Interface auto-refreshes. The default value is 30 sec.
+|nifi.nar.library.directory|The location of the nar library. The default value is ./lib and probably should be left as is.
+|nifi.nar.working.directory|The location of the nar working directory. The default value is ./work/nar and probably should be left as is.
+|nifi.documentation.working.directory|The documentation working directory. The default value is ./work/docs/components and probably should be left as is.
+|nifi.bored.yield.duration|When a component has no work to do (i.e., is "bored"), this is the amount of time it will wait before checking to see if it has new data to work on. This way, it does not use up CPU resources by checking for new work too often. When setting this property, be aware that it could add extra latency for components that do not constantly have work to do, as once they go into this "bored" state, they will wait this amount of time before checking for more work. The default value is 10 millis.
+|====
+
+*H2 Settings* +
+
+The H2 Settings section defines the settings for the H2 database, which keeps track of user access and flow controller history.
+
+|====
+|*Property*|*Description*
+|nifi.database.directory|The location of the H2 database directory. The default value is ./database_repository.
+|nifi.h2.url.append|This property specifies additional arguments to add to the connection string for the H2 database. The default value should be used and should not be changed. It is: ;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE.
+|====
+
+
+*FlowFile Repository* +
+
+The FlowFile repository keeps track of the attributes and current state of each FlowFile in the system. By default,
+this repository is installed in the same root installation directory as all the other repositories; however, it is advisable
+to configure it on a separate drive if available.
+
+|====
+|*Property*|*Description*
+|nifi.flowfile.repository.implementation|The FlowFile Repository implementation. The default value is org.apache.nifi.controller.repository.WriteAheadFlowFileRepository and should not be changed.
+|nifi.flowfile.repository.directory*|The location of the FlowFile Repository. The default value is ./flowfile_repository.
+|nifi.flowfile.repository.partitions|The number of partitions. The default value is 256.
+|nifi.flowfile.repository.checkpoint.interval| The FlowFile Repository checkpoint interval. The default value is 2 mins.
+|nifi.flowfile.repository.always.sync|If set to _true_, any change to the repository will be synchronized to the disk, meaning that NiFi will ask the operating system not to cache the information. This is very expensive and can significantly reduce NiFi performance. However, if it is _false_, there could be the potential for data loss if either there is a sudden power loss or the operating system crashes. The default value is _false_.
+|====
+
+*Swap Management* +
+
+NiFi keeps FlowFile information in memory (the JVM)
+but during surges of incoming data, the FlowFile information can start to take up so much of the JVM that system performance
+suffers. To counteract this effect, NiFi "swaps" the FlowFile information to disk temporarily until more JVM space becomes
+available again. These properties govern how that process occurs.
+
+|====
+|*Property*|*Description*
+|nifi.swap.manager.implementation|The Swap Manager implementation. The default value is org.apache.nifi.controller.FileSystemSwapManager and should not be changed.
+|nifi.queue.swap.threshold|The queue threshold at which NiFi starts to swap FlowFile information to disk. The default value is 20000.
+|nifi.swap.in.period|The swap in period. The default value is 5 sec.
+|nifi.swap.in.threads|The number of threads to use for swapping in. The default value is 1.
+|nifi.swap.out.period|The swap out period. The default value is 5 sec.
+|nifi.swap.out.threads|The number of threads to use for swapping out. The default value is 4.
+|====
+
+*Content Repository* +
+
+The Content Repository holds the content for all the FlowFiles in the system. By default, it is installed in the same root
+installation directory as all the other repositories; however, administrators will likely want to configure it on a separate
+drive if available. If nothing else, it is best if the Content Repository is not on the same drive as the FlowFile Repository.
+In dataflows that handle a large amount of data, the Content Repository could fill up a disk and the
+FlowFile Repository, if also on that disk, could become corrupt. To avoid this situation, configure these repositories on different drives.
+
+|====
+|*Property*|*Description*
+|nifi.content.repository.implementation|The Content Repository implementation. The default value is org.apache.nifi.controller.repository.FileSystemRepository and should not be changed.
+|nifi.content.claim.max.appendable.size|The maximum size for a content claim. The default value is 10 MB.
+|nifi.content.claim.max.flow.files|The maximum number of FlowFiles to assign to one content claim. The default value is 100.
+|nifi.content.repository.directory.default*|The location of the Content Repository. The default value is ./content_repository.
+|nifi.content.repository.archive.max.retention.period|If archiving is enabled (see nifi.content.repository.archive.enabled below), then
+this property specifies the maximum amount of time to keep the archived data. It is blank by default.
+|nifi.content.repository.archive.max.usage.percentage|If archiving is enabled (see nifi.content.repository.archive.enabled below), then this property also must have a value to indicate the maximum percentage of disk space that may be used for archiving. It is blank by default.
+|nifi.content.repository.archive.enabled|To enable archiving, set this to _true_ and specify a value for the nifi.content.repository.archive.max.usage.percentage property above. By default, archiving is not enabled.
+|nifi.content.repository.always.sync|If set to _true_, any change to the repository will be synchronized to the disk, meaning that NiFi will ask the operating system not to cache the information. This is very expensive and can significantly reduce NiFi performance. However, if it is _false_, there could be the potential for data loss if either there is a sudden power loss or the operating system crashes. The default value is _false_.
+|nifi.content.viewer.url|The URL for a web-based content viewer if one is available. It is blank by default.
+|====
+
+*Provenance Repository* +
+
+The Provenance Repository contains the information related to Data Provenance. The next three sections are for Provenance Repository properties.
+
+|====
+|*Property*|*Description*
+|nifi.provenance.repository.implementation|The Provenance Repository implementation. The default value is org.apache.nifi.provenance.PersistentProvenanceRepository and should not be changed.
+|====
+
+*Persistent Provenance Repository Properties* +
+
+|====
+|*Property*|*Description*
+|nifi.provenance.repository.directory.default*|The location of the Provenance Repository. The default value is ./provenance_repository.
+|nifi.provenance.repository.max.storage.time|The maximum amount of time to keep data provenance information. The default value is 24 hours.
+|nifi.provenance.repository.max.storage.size|The maximum amount of data provenance information to store at a time. The default is 1 GB.
+|nifi.provenance.repository.rollover.time|The amount of time to wait before rolling over the latest data provenance information so that it is available in the User Interface. The default value is 5 mins.
+|nifi.provenance.repository.rollover.size|The amount of information to roll over at a time. The default value is 100 MB.
+|nifi.provenance.repository.query.threads|The number of threads to use for Provenance Repository queries. The default value is 2.
+|nifi.provenance.repository.compress.on.rollover|Indicates whether to compress the provenance information when rolling it over. The default value is _true_.
+|nifi.provenance.repository.always.sync|If set to _true_, any change to the repository will be synchronized to the disk, meaning that NiFi will ask the operating system not to cache the information. This is very expensive and can significantly reduce NiFi performance. However, if it is _false_, there could be the potential for data loss if either there is a sudden power loss or the operating system crashes. The default value is _false_.
+|nifi.provenance.repository.journal.count|The number of journal files that should be used to serialize Provenance Event data. Increasing this value will allow more tasks to simultaneously update the repository but will result in more expensive merging of the journal files later. This value should ideally be equal to the number of threads that are expected to update the repository simultaneously, but 16 tends to work well in must environments. The default value is 16.
+|nifi.provenance.repository.indexed.fields|This is a comma-separated list of the fields that should be indexed and made searchable. Fields that are not indexed will not be searchable. Valid fields are: EventType, FlowFileUUID, Filename, TransitURI, ProcessorID, AlternateIdentifierURI, ContentType, Relationship, Details. The default value is: EventType, FlowFileUUID, Filename, ProcessorID.
+|nifi.provenance.repository.indexed.attributes|This is a comma-separated list of FlowFile Attributes that should be indexed and made searchable. It is blank by default.
+|nifi.provenance.repository.index.shard.size|Large values for the shard size will result in more Java heap usage when searching the Provenance Repository but should provide better performance. The default value is 500 MB.
+|====
+
+*Volatile Provenance Repository Properties* +
+
+|====
+|*Property*|*Description*
+|nifi.provenance.repository.buffer.size|The Provenance Repository buffer size. The default value is 100000.
+|====
+
+*Component Status Repository* +
+
+The Component Status Repository contains the information for the Component Status History tool in the User Interface. These
+properties govern how that tool works.
+
+|====
+|*Property*|*Description*
+|nifi.components.status.repository.implementation|The Component Status Repository implementation. The default value is org.apache.nifi.controller.status.history.VolatileComponentStatusRepository and should not be changed.
+|nifi.components.status.repository.buffer.size|Specifies the buffer size for the Component Status Repository. The default value is 288.
+|nifi.components.status.snapshot.frequency|This value indicates how often to present a snapshot of the components' status history. The default value is 5 mins.
+|====
+
+
+[[site_to_site_properties]]
+*Site to Site Properties* +
+
+These properties govern how this instance of NiFi communicates with remote instances of NiFi when Remote Process Groups are configured in the dataflow.
+
+|====
+|*Property*|*Description*
+|nifi.remote.input.socket.port|The remote input socket port for Site-to-Site communication. By default, it is blank, but it must have a value in order to use Remote Process Groups.
+|nifi.remote.input.secure|This indicates whether communication between this instance of NiFi and remote NiFi instances should be secure. By default, it is set to _true_. In order for secure site-to-site to work, many Security Properties (below) must also be configured.
+|====
+
+*Web Properties* +
+
+These properties pertain to the web-based User Interface.
+
+|====
+|*Property*|*Description*
+|nifi.web.war.directory|This is the location of the web war directory. The default value is ./lib.
+|nifi.web.http.host|The HTTP host. It is blank by default.
+|nifi.web.http.port|The HTTP port. The default value is 8080.
+|nifi.web.https.host|The HTTPS host. It is blank by default.
+|nifi.web.https.port|The HTTPS port. It is blank by default. When configuring NiFi to run securely, this port should be configured.
+|nif.web.jetty.working.directory|The location of the Jetty working directory. The default value is ./work/jetty.
+|nifi.web.jetty.threads|The number of Jetty threads. The default value is 200.
+|====
+
+*Security Properties* +
+
+These properties pertain to various security features in NiFi. Many of these properties are covered in more detail in the
+Security Configuration section of this Administrator's Guide.
+
+|====
+|*Property*|*Description*
+|nifi.sensitive.props.key|This is the password used to encrypt any sensitive property values that are configured in processors. By default, it is blank, but the system administrator should provide a value for it. It can be a string of any length. Be aware that once this password is set and one or more sensitive processor properties have been configured, this password should not be changed.
+|nifi.sensitive.props.algorithm|The algorithm used to encrypt sensitive properties. The default value is PBEWITHMD5AND256BITAES-CBC-OPENSSL.
+|nifi.sensitive.props.provider|The sensitive property provider. The default value is BC.
+|nifi.security.keystore*|The full path and name of the keystore. It is blank by default.
+|nifi.security.keystoreType|The keystore type. It is blank by default.
+|nifi.security.keystorePasswd|The keystore password. It is blank by default.
+|nifi.security.keyPasswd|The key password. It is blank by default.
+|nifi.security.truststore*|The full path and name of the truststore. It is blank by default.
+|nifi.security.truststoreType|The truststore type. It is blank by default.
+|nifi.security.truststorePasswd|The truststore password. It is blank by default.
+|nifi.security.needClientAuth|This indicates whether client authentication is required. It is blank by default.
+|nifi.security.user.credential.cache.duration|The length of time to cache user credentials. The default value is 24 hours.
+|nifi.security.user.authority.provider|This indicates what type of authority provider to use. The default value is file-provider, which refers to the file
+configured in the core property nifi.authority.provider.configuration.file. Another authority provider may be used, such as when the NiFi instance is part of a cluster. But the default value of file-provider is fine for a standalone instance of NiFi.
+|nifi.security.support.new.account.requests|This indicates whether a secure NiFi is configured to allow users to request access. It is blank by default.
+|nifi.security.ocsp.responder.url|This is the URL for the Online Certificate Status Protocol (OCSP) responder if one is being used. It is blank by default.
+|nifi.security.ocsp.responder.certificate|This is the location of the OCSP responder certificate if one is being used. It is blank by default.
+|====
+
+*Cluster Common Properties* +
+
+When setting up a NiFi cluster, these properties should be configured the same way on both the cluster manager and the nodes.
+
+|====
+|*Property*|*Description*
+|nifi.cluster.protocol.heartbeat.interval|The interval at which nodes should emit heartbeats to the cluster manager. The default value is 5 sec.
+|nifi.cluster.protocol.is.secure|This indicates whether cluster communications are secure. The default value is _false_.
+|nifi.cluster.protocol.socket.timeout|The amount of time to wait for a cluster protocol socket to be established before trying again. The default value is 30 sec.
+|nifi.cluster.protocol.connection.handshake.timeout|The amount of time to wait for a node to connect to the cluster. The default value is 45 sec.
+|====
+
+*Multicast Cluster Common Properties* +
+If multicast is used, the following nifi.cluster.protocol.multicast.xxx properties must be configured. By default, unicast is used.
+
+|====
+|*Property*|*Description*
+|nifi.cluster.protocol.use.multicast|Indicates whether multicast is being used. The default value is _false_.
+|nifi.cluster.protocol.multicast.address|The cluster multicast address. It is blank by default.
+|nifi.cluster.protocol.multicast.port|The cluster multicast port. It is blank by default.
+|nifi.cluster.protocol.multicast.service.broadcast.delay|The multicast service broadcast delay. The default value is 500 ms.
+|nifi.cluster.protocol.multicast.service.locator.attempts|The number of multicast service locator attempts to make. The default value is 3.
+|nifi.cluster.protocol.multicast.service.locator.attempts.delay|The multicast service locator attempts delay. The default value is 1 sec.
+|====
+
+*Cluster Node Properties* +
+
+Only configure these properties for cluster nodes.
+
+|====
+|*Property*|*Description*
+|nifi.cluster.is.node|Set this to _true_ if the instance is a node in a cluster. The default value is _false_.
+|nifi.cluster.node.address|The fully qualified address of the node. It is blank by default.
+|nifi.cluster.node.protocol.port|The node's protocol port. It is blank by default.
+|nifi.cluster.node.protocol.threads|The number of threads used for the node protocol. The default value is 2.
+|nifi.cluster.node.unicast.manager.address|If multicast is not used, the value for this property should be the same as the value configured on the cluster manager for manager address.
+|nifi.cluster.node.unicast.manager.protocol.port|If multicast is not used, the value for this property should be the same as the value configured on the cluster manager for manager protocol port.
+|====
+
+*Cluster Manager Properties* +
+
+Only configure these properties for the cluster manager.
+
+|====
+|*Property*|*Description*
+|nifi.cluster.is.manager|Set this to _true_ if the instance is a cluster manager. The default value is _false_.
+|nifi.cluster.manager.address|The fully qualified address of the cluster manager. It is blank by default.
+|nifi.cluster.manager.protocol.port|The cluster manager's protocol port. It is blank by default.
+|nifi.cluster.manager.node.firewall.file|The location of the node firewall file. This is a file that may be used to list all the nodes that are allowed to connect to the cluster. It provides an additional layer of security. This value is blank by default.
+|nifi.cluster.manager.node.event.history.size|The size of the cluster manager's event history. The default value is 10.
+|nifi.cluster.manager.node.api.connection.timeout|The amount of time to wait for an API connection to be made. The default value is 30 sec.
+|nifi.cluster.manager.node.api.read.timeout|The API read timeout. The default value is 30 sec.
+|nifi.cluster.manager.node.api.request.threads|The number of threads to use for API requests. The default value is 10.
+|nifi.cluster.manager.flow.retrieval.delay|The delay before the cluster manager retrieves the latest flow configuration. The default value is 5 sec.
+|nifi.cluster.manager.protocol.threads|The number of threads used for the cluster manager protocol. The default value is 10.
+|nifi.cluster.manager.safemode.duration|Upon restart of an already existing cluster, this is the amount of time that the cluster manager waits for the primary node to connect before giving up and selecting another node to be the primary node. The default value is 0 sec, which means to wait forever. If the administrator does not care which node is the primary node, this value can be changed to some amount of time other than 0 sec.
+|====
+
+
+NOTE: *For Upgrading* - Take care when configuring the properties above that are marked with an asterisk (*). To make the upgrade process easier, it is advisable to change the default configurations to locations outside the main root installation directory. In this way, these items can remain in their configured location through an upgrade, and NiFi can find all the repositories and configuration files and pick up where it left off as soon as the old version is stopped and the new version is started. Furthermore, the administrator may reuse this _nifi.properties_ file and any other configuration files without having to re-configure them each time an upgrade takes place. As previously noted, it is important to check for any changes in the _nifi.properties_ file of the new version when upgrading and make sure they are reflected in the _nifi.properties_ file you use.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
[09/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-nar/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-nar/pom.xml b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-nar/pom.xml
new file mode 100644
index 0000000..77fbe97
--- /dev/null
+++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-nar/pom.xml
@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-aws-bundle</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-aws-nar</artifactId>
+ <packaging>nar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-aws-processors</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-nar/src/main/resources/META-INF/NOTICE
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-nar/src/main/resources/META-INF/NOTICE b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-nar/src/main/resources/META-INF/NOTICE
new file mode 100644
index 0000000..64d4975
--- /dev/null
+++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-nar/src/main/resources/META-INF/NOTICE
@@ -0,0 +1,74 @@
+nifi-aws-nar
+Copyright 2015 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+******************
+Apache Software License v2
+******************
+
+The following binary components are provided under the Apache Software License v2
+
+ (ASLv2) Apache HttpComponents
+ The following NOTICE information applies:
+ Apache HttpClient
+ Copyright 1999-2014 The Apache Software Foundation
+
+ Apache HttpCore
+ Copyright 2005-2014 The Apache Software Foundation
+
+ This project contains annotations derived from JCIP-ANNOTATIONS
+ Copyright (c) 2005 Brian Goetz and Tim Peierls. See http://www.jcip.net
+
+ (ASLv2) Joda Time
+ The following NOTICE information applies:
+ This product includes software developed by
+ Joda.org (http://www.joda.org/).
+
+ (ASLv2) Apache Commons Codec
+ The following NOTICE information applies:
+ Apache Commons Codec
+ Copyright 2002-2014 The Apache Software Foundation
+
+ src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java
+ contains test data from http://aspell.net/test/orig/batch0.tab.
+ Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org)
+
+ ===============================================================================
+
+ The content of package org.apache.commons.codec.language.bm has been translated
+ from the original php source code available at http://stevemorse.org/phoneticinfo.htm
+ with permission from the original authors.
+ Original source copyright:
+ Copyright (c) 2008 Alexander Beider & Stephen P. Morse.
+
+ (ASLv2) Apache Commons Logging
+ The following NOTICE information applies:
+ Apache Commons Logging
+ Copyright 2003-2013 The Apache Software Foundation
+
+ (ASLv2) Apache Commons Lang
+ The following NOTICE information applies:
+ Apache Commons Lang
+ Copyright 2001-2014 The Apache Software Foundation
+
+ This product includes software from the Spring Framework,
+ under the Apache License 2.0 (see: StringUtils.containsWhitespace())
+
+ (ASLv2) Amazon Web Services SDK
+ The following NOTICE information applies:
+ Copyright 2010-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
+ This product includes software developed by
+ Amazon Technologies, Inc (http://www.amazon.com/).
+
+ **********************
+ THIRD PARTY COMPONENTS
+ **********************
+ This software includes third party software subject to the following copyrights:
+ - XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty.
+ - JSON parsing and utility functions from JSON.org - Copyright 2002 JSON.org.
+ - PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc.
+
+
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/pom.xml b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/pom.xml
new file mode 100644
index 0000000..b784524
--- /dev/null
+++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/pom.xml
@@ -0,0 +1,67 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-aws-bundle</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>nifi-aws-processors</artifactId>
+ <packaging>jar</packaging>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-processor-utils</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-mock</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-simple</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes combine.children="append">
+ <exclude>src/test/resources/hello.txt</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/AbstractAWSProcessor.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/AbstractAWSProcessor.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/AbstractAWSProcessor.java
new file mode 100644
index 0000000..a781ff9
--- /dev/null
+++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/AbstractAWSProcessor.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.aws;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.AllowableValue;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+
+import com.amazonaws.AmazonWebServiceClient;
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AnonymousAWSCredentials;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.auth.PropertiesCredentials;
+import com.amazonaws.regions.Region;
+import com.amazonaws.regions.Regions;
+
+public abstract class AbstractAWSProcessor<ClientType extends AmazonWebServiceClient> extends AbstractProcessor {
+
+ public static final Relationship REL_SUCCESS = new Relationship.Builder().name("success")
+ .description("FlowFiles are routed to success after being successfully copied to Amazon S3").build();
+ public static final Relationship REL_FAILURE = new Relationship.Builder().name("failure")
+ .description("FlowFiles are routed to failure if unable to be copied to Amazon S3").build();
+
+ public static final Set<Relationship> relationships = Collections.unmodifiableSet(
+ new HashSet<>(Arrays.asList(REL_SUCCESS, REL_FAILURE)));
+
+ public static final PropertyDescriptor CREDENTAILS_FILE = new PropertyDescriptor.Builder()
+ .name("Credentials File")
+ .expressionLanguageSupported(false)
+ .required(false)
+ .addValidator(StandardValidators.FILE_EXISTS_VALIDATOR)
+ .build();
+ public static final PropertyDescriptor ACCESS_KEY = new PropertyDescriptor.Builder()
+ .name("Access Key")
+ .expressionLanguageSupported(false)
+ .required(false)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .sensitive(true)
+ .build();
+ public static final PropertyDescriptor SECRET_KEY = new PropertyDescriptor.Builder()
+ .name("Secret Key")
+ .expressionLanguageSupported(false)
+ .required(false)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .sensitive(true)
+ .build();
+ public static final PropertyDescriptor REGION = new PropertyDescriptor.Builder()
+ .name("Region")
+ .required(true)
+ .allowableValues(getAvailableRegions())
+ .defaultValue(createAllowableValue(Regions.DEFAULT_REGION).getValue())
+ .build();
+
+ public static final PropertyDescriptor TIMEOUT = new PropertyDescriptor.Builder()
+ .name("Communications Timeout")
+ .required(true)
+ .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR)
+ .defaultValue("30 secs")
+ .build();
+
+ private volatile ClientType client;
+
+ private static AllowableValue createAllowableValue(final Regions regions) {
+ return new AllowableValue(regions.getName(), regions.getName(), regions.getName());
+ }
+
+ private static AllowableValue[] getAvailableRegions() {
+ final List<AllowableValue> values = new ArrayList<>();
+ for (final Regions regions : Regions.values()) {
+ values.add(createAllowableValue(regions));
+ }
+
+ return (AllowableValue[]) values.toArray(new AllowableValue[values.size()]);
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return relationships;
+ }
+
+ @Override
+ protected Collection<ValidationResult> customValidate(final ValidationContext validationContext) {
+ final List<ValidationResult> problems = new ArrayList<>(super.customValidate(validationContext));
+
+ final boolean accessKeySet = validationContext.getProperty(ACCESS_KEY).isSet();
+ final boolean secretKeySet = validationContext.getProperty(SECRET_KEY).isSet();
+ if ((accessKeySet && !secretKeySet) || (secretKeySet && !accessKeySet)) {
+ problems.add(new ValidationResult.Builder().input("Access Key").valid(false).explanation("If setting Secret Key or Access Key, must set both").build());
+ }
+
+ final boolean credentialsFileSet = validationContext.getProperty(CREDENTAILS_FILE).isSet();
+ if ((secretKeySet || accessKeySet) && credentialsFileSet) {
+ problems.add(new ValidationResult.Builder().input("Access Key").valid(false).explanation("Cannot set both Credentials File and Secret Key/Access Key").build());
+ }
+
+ return problems;
+ }
+
+ protected ClientConfiguration createConfiguration(final ProcessContext context) {
+ final ClientConfiguration config = new ClientConfiguration();
+ config.setMaxConnections(context.getMaxConcurrentTasks());
+ config.setMaxErrorRetry(0);
+ config.setUserAgent("NiFi");
+
+ final int commsTimeout = context.getProperty(TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue();
+ config.setConnectionTimeout(commsTimeout);
+ config.setSocketTimeout(commsTimeout);
+
+ return config;
+ }
+
+ @OnScheduled
+ public void onScheduled(final ProcessContext context) {
+ final ClientType awsClient = createClient(context, getCredentials(context), createConfiguration(context));
+ this.client = awsClient;
+
+ // if the processor supports REGION, get the configured region.
+ if (getSupportedPropertyDescriptors().contains(REGION)) {
+ final String region = context.getProperty(REGION).getValue();
+ if (region != null) {
+ client.setRegion(Region.getRegion(Regions.fromName(region)));
+ }
+ }
+ }
+
+ protected abstract ClientType createClient(final ProcessContext context, final AWSCredentials credentials, final ClientConfiguration config);
+
+ protected ClientType getClient() {
+ return client;
+ }
+
+ protected AWSCredentials getCredentials(final ProcessContext context) {
+ final String accessKey = context.getProperty(ACCESS_KEY).getValue();
+ final String secretKey = context.getProperty(SECRET_KEY).getValue();
+
+ final String credentialsFile = context.getProperty(CREDENTAILS_FILE).getValue();
+
+ if (credentialsFile != null) {
+ try {
+ return new PropertiesCredentials(new File(credentialsFile));
+ } catch (final IOException ioe) {
+ throw new ProcessException("Could not read Credentials File", ioe);
+ }
+ }
+
+ if (accessKey != null && secretKey != null) {
+ return new BasicAWSCredentials(accessKey, secretKey);
+ }
+
+ return new AnonymousAWSCredentials();
+ }
+
+ protected boolean isEmpty(final String value) {
+ return value == null || value.trim().equals("");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/AbstractS3Processor.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/AbstractS3Processor.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/AbstractS3Processor.java
new file mode 100644
index 0000000..76880ef
--- /dev/null
+++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/AbstractS3Processor.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.aws.s3;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.processors.aws.AbstractAWSProcessor;
+
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.model.AccessControlList;
+import com.amazonaws.services.s3.model.CanonicalGrantee;
+import com.amazonaws.services.s3.model.EmailAddressGrantee;
+import com.amazonaws.services.s3.model.Grantee;
+import com.amazonaws.services.s3.model.Owner;
+import com.amazonaws.services.s3.model.Permission;
+
+public abstract class AbstractS3Processor extends AbstractAWSProcessor<AmazonS3Client> {
+
+ public static final PropertyDescriptor FULL_CONTROL_USER_LIST = new PropertyDescriptor.Builder()
+ .name("FullControl User List")
+ .required(false)
+ .expressionLanguageSupported(true)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .description("A comma-separated list of Amazon User ID's or E-mail addresses that specifies who should have Full Control for an object")
+ .defaultValue("${s3.permissions.full.users}")
+ .build();
+ public static final PropertyDescriptor READ_USER_LIST = new PropertyDescriptor.Builder()
+ .name("Read Permission User List")
+ .required(false)
+ .expressionLanguageSupported(true)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .description("A comma-separated list of Amazon User ID's or E-mail addresses that specifies who should have Read Access for an object")
+ .defaultValue("${s3.permissions.read.users}")
+ .build();
+ public static final PropertyDescriptor WRITE_USER_LIST = new PropertyDescriptor.Builder()
+ .name("Write Permission User List")
+ .required(false)
+ .expressionLanguageSupported(true)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .description("A comma-separated list of Amazon User ID's or E-mail addresses that specifies who should have Write Access for an object")
+ .defaultValue("${s3.permissions.write.users}")
+ .build();
+ public static final PropertyDescriptor READ_ACL_LIST = new PropertyDescriptor.Builder()
+ .name("Read ACL User List")
+ .required(false)
+ .expressionLanguageSupported(true)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .description("A comma-separated list of Amazon User ID's or E-mail addresses that specifies who should have permissions to read the Access Control List for an object")
+ .defaultValue("${s3.permissions.readacl.users}")
+ .build();
+ public static final PropertyDescriptor WRITE_ACL_LIST = new PropertyDescriptor.Builder()
+ .name("Write ACL User List")
+ .required(false)
+ .expressionLanguageSupported(true)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .description("A comma-separated list of Amazon User ID's or E-mail addresses that specifies who should have permissions to change the Access Control List for an object")
+ .defaultValue("${s3.permissions.writeacl.users}")
+ .build();
+ public static final PropertyDescriptor OWNER = new PropertyDescriptor.Builder()
+ .name("Owner")
+ .required(false)
+ .expressionLanguageSupported(true)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .description("The Amazon ID to use for the object's owner")
+ .defaultValue("${s3.owner}")
+ .build();
+ public static final PropertyDescriptor BUCKET = new PropertyDescriptor.Builder()
+ .name("Bucket")
+ .expressionLanguageSupported(true)
+ .required(true)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+ public static final PropertyDescriptor KEY = new PropertyDescriptor.Builder()
+ .name("Object Key")
+ .required(true)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .expressionLanguageSupported(true)
+ .defaultValue("${filename}")
+ .build();
+
+ @Override
+ protected AmazonS3Client createClient(final ProcessContext context, final AWSCredentials credentials, final ClientConfiguration config) {
+ return new AmazonS3Client(credentials, config);
+ }
+
+ protected Grantee createGrantee(final String value) {
+ if (isEmpty(value)) {
+ return null;
+ }
+
+ if (value.contains("@")) {
+ return new EmailAddressGrantee(value);
+ } else {
+ return new CanonicalGrantee(value);
+ }
+ }
+
+ protected final List<Grantee> createGrantees(final String value) {
+ if (isEmpty(value)) {
+ return Collections.emptyList();
+ }
+
+ final List<Grantee> grantees = new ArrayList<>();
+ final String[] vals = value.split(",");
+ for (final String val : vals) {
+ final String identifier = val.trim();
+ final Grantee grantee = createGrantee(identifier);
+ if (grantee != null) {
+ grantees.add(grantee);
+ }
+ }
+ return grantees;
+ }
+
+ protected final AccessControlList createACL(final ProcessContext context, final FlowFile flowFile) {
+ final AccessControlList acl = new AccessControlList();
+
+ final String ownerId = context.getProperty(OWNER).evaluateAttributeExpressions(flowFile).getValue();
+ if (!isEmpty(ownerId)) {
+ final Owner owner = new Owner();
+ owner.setId(ownerId);
+ acl.setOwner(owner);
+ }
+
+ for (final Grantee grantee : createGrantees(context.getProperty(FULL_CONTROL_USER_LIST).evaluateAttributeExpressions(flowFile).getValue())) {
+ acl.grantPermission(grantee, Permission.FullControl);
+ }
+
+ for (final Grantee grantee : createGrantees(context.getProperty(READ_USER_LIST).evaluateAttributeExpressions(flowFile).getValue())) {
+ acl.grantPermission(grantee, Permission.Read);
+ }
+
+ for (final Grantee grantee : createGrantees(context.getProperty(WRITE_USER_LIST).evaluateAttributeExpressions(flowFile).getValue())) {
+ acl.grantPermission(grantee, Permission.Write);
+ }
+
+ for (final Grantee grantee : createGrantees(context.getProperty(READ_ACL_LIST).evaluateAttributeExpressions(flowFile).getValue())) {
+ acl.grantPermission(grantee, Permission.ReadAcp);
+ }
+
+ for (final Grantee grantee : createGrantees(context.getProperty(WRITE_ACL_LIST).evaluateAttributeExpressions(flowFile).getValue())) {
+ acl.grantPermission(grantee, Permission.WriteAcp);
+ }
+
+ return acl;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/FetchS3Object.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/FetchS3Object.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/FetchS3Object.java
new file mode 100644
index 0000000..2406b67
--- /dev/null
+++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/FetchS3Object.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.aws.s3;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.SeeAlso;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.util.StandardValidators;
+
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.GetObjectRequest;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.amazonaws.services.s3.model.S3Object;
+
+@SupportsBatching
+@SeeAlso({PutS3Object.class})
+@Tags({"Amazon", "S3", "AWS", "Get", "Fetch"})
+@CapabilityDescription("Retrieves the contents of an S3 Object and writes it to the content of a FlowFile")
+@WritesAttributes({
+ @WritesAttribute(attribute = "s3.bucket", description = "The name of the S3 bucket"),
+ @WritesAttribute(attribute = "path", description = "The path of the file"),
+ @WritesAttribute(attribute = "absolute.path", description = "The path of the file"),
+ @WritesAttribute(attribute = "filename", description = "The name of the file"),
+ @WritesAttribute(attribute = "hash.value", description = "The MD5 sum of the file"),
+ @WritesAttribute(attribute = "hash.algorithm", description = "MD5"),
+ @WritesAttribute(attribute = "mime.type", description = "If S3 provides the content type/MIME type, this attribute will hold that file"),
+ @WritesAttribute(attribute = "s3.etag", description = "The ETag that can be used to see if the file has changed"),
+ @WritesAttribute(attribute = "s3.expirationTime", description = "If the file has an expiration date, this attribute will be set, containing the milliseconds since epoch in UTC time"),
+ @WritesAttribute(attribute = "s3.expirationTimeRuleId", description = "The ID of the rule that dictates this object's expiration time"),
+ @WritesAttribute(attribute = "s3.version", description = "The version of the S3 object"),})
+public class FetchS3Object extends AbstractS3Processor {
+
+ public static final PropertyDescriptor VERSION_ID = new PropertyDescriptor.Builder()
+ .name("Version")
+ .description("The Version of the Object to download")
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .expressionLanguageSupported(true)
+ .required(false)
+ .build();
+
+ public static final List<PropertyDescriptor> properties = Collections.unmodifiableList(
+ Arrays.asList(BUCKET, KEY, REGION, ACCESS_KEY, SECRET_KEY, CREDENTAILS_FILE, TIMEOUT, VERSION_ID));
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return properties;
+ }
+
+ @Override
+ public void onTrigger(final ProcessContext context, final ProcessSession session) {
+ FlowFile flowFile = session.get();
+ if (flowFile == null) {
+ return;
+ }
+
+ final long startNanos = System.nanoTime();
+ final String bucket = context.getProperty(BUCKET).evaluateAttributeExpressions(flowFile).getValue();
+ final String key = context.getProperty(KEY).evaluateAttributeExpressions(flowFile).getValue();
+ final String versionId = context.getProperty(VERSION_ID).evaluateAttributeExpressions(flowFile).getValue();
+
+ final AmazonS3 client = getClient();
+ final GetObjectRequest request;
+ if (versionId == null) {
+ request = new GetObjectRequest(bucket, key);
+ } else {
+ request = new GetObjectRequest(bucket, key, versionId);
+ }
+
+ final Map<String, String> attributes = new HashMap<>();
+ try (final S3Object s3Object = client.getObject(request)) {
+ flowFile = session.importFrom(s3Object.getObjectContent(), flowFile);
+ attributes.put("s3.bucket", s3Object.getBucketName());
+
+ final ObjectMetadata metadata = s3Object.getObjectMetadata();
+ if (metadata.getContentDisposition() != null) {
+ final String fullyQualified = metadata.getContentDisposition();
+ final int lastSlash = fullyQualified.lastIndexOf("/");
+ if (lastSlash > -1 && lastSlash < fullyQualified.length() - 1) {
+ attributes.put(CoreAttributes.PATH.key(), fullyQualified.substring(0, lastSlash));
+ attributes.put(CoreAttributes.ABSOLUTE_PATH.key(), fullyQualified);
+ attributes.put(CoreAttributes.FILENAME.key(), fullyQualified.substring(lastSlash + 1));
+ } else {
+ attributes.put(CoreAttributes.FILENAME.key(), metadata.getContentDisposition());
+ }
+ }
+ if (metadata.getContentMD5() != null) {
+ attributes.put("hash.value", metadata.getContentMD5());
+ attributes.put("hash.algorithm", "MD5");
+ }
+ if (metadata.getContentType() != null) {
+ attributes.put(CoreAttributes.MIME_TYPE.key(), metadata.getContentType());
+ }
+ if (metadata.getETag() != null) {
+ attributes.put("s3.etag", metadata.getETag());
+ }
+ if (metadata.getExpirationTime() != null) {
+ attributes.put("s3.expirationTime", String.valueOf(metadata.getExpirationTime().getTime()));
+ }
+ if (metadata.getExpirationTimeRuleId() != null) {
+ attributes.put("s3.expirationTimeRuleId", metadata.getExpirationTimeRuleId());
+ }
+ if (metadata.getUserMetadata() != null) {
+ attributes.putAll(metadata.getUserMetadata());
+ }
+ if (metadata.getVersionId() != null) {
+ attributes.put("s3.version", metadata.getVersionId());
+ }
+ } catch (final IOException | AmazonClientException ioe) {
+ getLogger().error("Failed to retrieve S3 Object for {}; routing to failure", new Object[]{flowFile, ioe});
+ session.transfer(flowFile, REL_FAILURE);
+ return;
+ }
+
+ if (!attributes.isEmpty()) {
+ flowFile = session.putAllAttributes(flowFile, attributes);
+ }
+
+ session.transfer(flowFile, REL_SUCCESS);
+ final long transferMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
+ getLogger().info("Successfully retrieved S3 Object for {} in {} millis; routing to success", new Object[]{flowFile, transferMillis});
+ session.getProvenanceReporter().receive(flowFile, "http://" + bucket + ".amazonaws.com/" + key, transferMillis);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/PutS3Object.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/PutS3Object.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/PutS3Object.java
new file mode 100644
index 0000000..24c82dd
--- /dev/null
+++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/PutS3Object.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.aws.s3;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.nifi.annotation.behavior.DynamicProperty;
+import org.apache.nifi.annotation.behavior.ReadsAttribute;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.SeeAlso;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.io.InputStreamCallback;
+import org.apache.nifi.processor.util.StandardValidators;
+
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.AccessControlList;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.amazonaws.services.s3.model.PutObjectRequest;
+import com.amazonaws.services.s3.model.PutObjectResult;
+import com.amazonaws.services.s3.model.StorageClass;
+
+@SupportsBatching
+@SeeAlso({FetchS3Object.class})
+@Tags({"Amazon", "S3", "AWS", "Archive", "Put"})
+@CapabilityDescription("Puts FlowFiles to an Amazon S3 Bucket")
+@DynamicProperty(name = "The name of a User-Defined Metadata field to add to the S3 Object",
+ value = "The value of a User-Defined Metadata field to add to the S3 Object",
+ description = "Allows user-defined metadata to be added to the S3 object as key/value pairs",
+ supportsExpressionLanguage = true)
+@ReadsAttribute(attribute = "filename", description = "Uses the FlowFile's filename as the filename for the S3 object")
+@WritesAttributes({
+ @WritesAttribute(attribute = "s3.version", description = "The version of the S3 Object that was put to S3"),
+ @WritesAttribute(attribute = "s3.etag", description = "The ETag of the S3 Object"),
+ @WritesAttribute(attribute = "s3.expiration", description = "A human-readable form of the expiration date of the S3 object, if one is set")
+})
+public class PutS3Object extends AbstractS3Processor {
+
+ public static final PropertyDescriptor EXPIRATION_RULE_ID = new PropertyDescriptor.Builder()
+ .name("Expiration Time Rule")
+ .required(false)
+ .expressionLanguageSupported(true)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+
+ public static final PropertyDescriptor STORAGE_CLASS = new PropertyDescriptor.Builder()
+ .name("Storage Class")
+ .required(true)
+ .allowableValues(StorageClass.Standard.name(), StorageClass.ReducedRedundancy.name())
+ .defaultValue(StorageClass.Standard.name())
+ .build();
+
+ public static final List<PropertyDescriptor> properties = Collections.unmodifiableList(
+ Arrays.asList(KEY, BUCKET, ACCESS_KEY, SECRET_KEY, CREDENTAILS_FILE, STORAGE_CLASS, REGION, TIMEOUT, EXPIRATION_RULE_ID,
+ FULL_CONTROL_USER_LIST, READ_USER_LIST, WRITE_USER_LIST, READ_ACL_LIST, WRITE_ACL_LIST, OWNER));
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return properties;
+ }
+
+ @Override
+ protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) {
+ return new PropertyDescriptor.Builder()
+ .name(propertyDescriptorName)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .expressionLanguageSupported(true)
+ .dynamic(true)
+ .build();
+ }
+
+ public void onTrigger(final ProcessContext context, final ProcessSession session) {
+ FlowFile flowFile = session.get();
+ if (flowFile == null) {
+ return;
+ }
+
+ final long startNanos = System.nanoTime();
+
+ final String bucket = context.getProperty(BUCKET).evaluateAttributeExpressions(flowFile).getValue();
+ final String key = context.getProperty(KEY).evaluateAttributeExpressions(flowFile).getValue();
+
+ final AmazonS3 s3 = getClient();
+ final FlowFile ff = flowFile;
+ final Map<String, String> attributes = new HashMap<>();
+ try {
+ session.read(flowFile, new InputStreamCallback() {
+ @Override
+ public void process(final InputStream rawIn) throws IOException {
+ try (final InputStream in = new BufferedInputStream(rawIn)) {
+ final ObjectMetadata objectMetadata = new ObjectMetadata();
+ objectMetadata.setContentDisposition(ff.getAttribute(CoreAttributes.FILENAME.key()));
+ objectMetadata.setContentLength(ff.getSize());
+
+ final String expirationRule = context.getProperty(EXPIRATION_RULE_ID).evaluateAttributeExpressions(ff).getValue();
+ if (expirationRule != null) {
+ objectMetadata.setExpirationTimeRuleId(expirationRule);
+ }
+
+ final Map<String, String> userMetadata = new HashMap<>();
+ for (final Map.Entry<PropertyDescriptor, String> entry : context.getProperties().entrySet()) {
+ if (entry.getKey().isDynamic()) {
+ final String value = context.getProperty(entry.getKey()).evaluateAttributeExpressions(ff).getValue();
+ userMetadata.put(entry.getKey().getName(), value);
+ }
+ }
+
+ if (!userMetadata.isEmpty()) {
+ objectMetadata.setUserMetadata(userMetadata);
+ }
+
+ final PutObjectRequest request = new PutObjectRequest(bucket, key, in, objectMetadata);
+ request.setStorageClass(StorageClass.valueOf(context.getProperty(STORAGE_CLASS).getValue()));
+ final AccessControlList acl = createACL(context, ff);
+ if (acl != null) {
+ request.setAccessControlList(acl);
+ }
+
+ final PutObjectResult result = s3.putObject(request);
+ if (result.getVersionId() != null) {
+ attributes.put("s3.version", result.getVersionId());
+ }
+
+ attributes.put("s3.etag", result.getETag());
+
+ final Date expiration = result.getExpirationTime();
+ if (expiration != null) {
+ attributes.put("s3.expiration", expiration.toString());
+ }
+ }
+ }
+ });
+
+ if (!attributes.isEmpty()) {
+ flowFile = session.putAllAttributes(flowFile, attributes);
+ }
+ session.transfer(flowFile, REL_SUCCESS);
+
+ final String url = "http://" + bucket + ".s3.amazonaws.com/" + key;
+ final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
+ session.getProvenanceReporter().send(flowFile, url, millis);
+
+ getLogger().info("Successfully put {} to Amazon S3 in {} milliseconds", new Object[]{ff, millis});
+ } catch (final ProcessException | AmazonClientException pe) {
+ getLogger().error("Failed to put {} to Amazon S3 due to {}", new Object[]{flowFile, pe});
+ session.transfer(flowFile, REL_FAILURE);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/sns/AbstractSNSProcessor.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/sns/AbstractSNSProcessor.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/sns/AbstractSNSProcessor.java
new file mode 100644
index 0000000..5b57647
--- /dev/null
+++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/sns/AbstractSNSProcessor.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.aws.sns;
+
+import org.apache.nifi.components.AllowableValue;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.processors.aws.AbstractAWSProcessor;
+
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.services.sns.AmazonSNSClient;
+
+public abstract class AbstractSNSProcessor extends AbstractAWSProcessor<AmazonSNSClient> {
+
+ protected static final AllowableValue ARN_TYPE_TOPIC
+ = new AllowableValue("Topic ARN", "Topic ARN", "The ARN is the name of a topic");
+ protected static final AllowableValue ARN_TYPE_TARGET
+ = new AllowableValue("Target ARN", "Target ARN", "The ARN is the name of a particular Target, used to notify a specific subscriber");
+
+ public static final PropertyDescriptor ARN = new PropertyDescriptor.Builder()
+ .name("Amazon Resource Name (ARN)")
+ .description("The name of the resource to which notifications should be published")
+ .expressionLanguageSupported(true)
+ .required(true)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+
+ public static final PropertyDescriptor ARN_TYPE = new PropertyDescriptor.Builder()
+ .name("ARN Type")
+ .description("The type of Amazon Resource Name that is being used.")
+ .expressionLanguageSupported(false)
+ .required(true)
+ .allowableValues(ARN_TYPE_TOPIC, ARN_TYPE_TARGET)
+ .defaultValue(ARN_TYPE_TOPIC.getValue())
+ .build();
+
+ @Override
+ protected AmazonSNSClient createClient(final ProcessContext context, final AWSCredentials credentials, final ClientConfiguration config) {
+ return new AmazonSNSClient(credentials, config);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/sns/PutSNS.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/sns/PutSNS.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/sns/PutSNS.java
new file mode 100644
index 0000000..7d42703
--- /dev/null
+++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/sns/PutSNS.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.aws.sns;
+
+import java.io.ByteArrayOutputStream;
+import java.nio.charset.Charset;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.SeeAlso;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.processors.aws.sqs.GetSQS;
+import org.apache.nifi.processors.aws.sqs.PutSQS;
+
+import com.amazonaws.services.sns.AmazonSNSClient;
+import com.amazonaws.services.sns.model.MessageAttributeValue;
+import com.amazonaws.services.sns.model.PublishRequest;
+
+@SupportsBatching
+@SeeAlso({GetSQS.class, PutSQS.class})
+@Tags({"amazon", "aws", "sns", "topic", "put", "publish", "pubsub"})
+@CapabilityDescription("Sends the content of a FlowFile as a notification to the Amazon Simple Notification Service")
+public class PutSNS extends AbstractSNSProcessor {
+
+ public static final PropertyDescriptor CHARACTER_ENCODING = new PropertyDescriptor.Builder()
+ .name("Character Set")
+ .description("The character set in which the FlowFile's content is encoded")
+ .defaultValue("UTF-8")
+ .expressionLanguageSupported(true)
+ .addValidator(StandardValidators.CHARACTER_SET_VALIDATOR)
+ .required(true)
+ .build();
+ public static final PropertyDescriptor USE_JSON_STRUCTURE = new PropertyDescriptor.Builder()
+ .name("Use JSON Structure")
+ .description("If true, the contents of the FlowFile must be JSON with a top-level element named 'default'."
+ + " Additional elements can be used to send different messages to different protocols. See the Amazon"
+ + " SNS Documentation for more information.")
+ .defaultValue("false")
+ .allowableValues("true", "false")
+ .required(true)
+ .build();
+ public static final PropertyDescriptor SUBJECT = new PropertyDescriptor.Builder()
+ .name("E-mail Subject")
+ .description("The optional subject to use for any subscribers that are subscribed via E-mail")
+ .expressionLanguageSupported(true)
+ .required(false)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+
+ public static final List<PropertyDescriptor> properties = Collections.unmodifiableList(
+ Arrays.asList(ARN, ARN_TYPE, SUBJECT, REGION, ACCESS_KEY, SECRET_KEY, CREDENTAILS_FILE, TIMEOUT,
+ USE_JSON_STRUCTURE, CHARACTER_ENCODING));
+
+ public static final int MAX_SIZE = 256 * 1024;
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return properties;
+ }
+
+ @Override
+ protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) {
+ return new PropertyDescriptor.Builder()
+ .name(propertyDescriptorName)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .expressionLanguageSupported(true)
+ .required(false)
+ .dynamic(true)
+ .build();
+ }
+
+ @Override
+ public void onTrigger(final ProcessContext context, final ProcessSession session) {
+ FlowFile flowFile = session.get();
+ if (flowFile == null) {
+ return;
+ }
+
+ if (flowFile.getSize() > MAX_SIZE) {
+ getLogger().error("Cannot publish {} to SNS because its size exceeds Amazon SNS's limit of 256KB; routing to failure", new Object[]{flowFile});
+ session.transfer(flowFile, REL_FAILURE);
+ return;
+ }
+
+ final Charset charset = Charset.forName(context.getProperty(CHARACTER_ENCODING).evaluateAttributeExpressions(flowFile).getValue());
+
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ session.exportTo(flowFile, baos);
+ final String message = new String(baos.toByteArray(), charset);
+
+ final AmazonSNSClient client = getClient();
+ final PublishRequest request = new PublishRequest();
+ request.setMessage(message);
+
+ if (context.getProperty(USE_JSON_STRUCTURE).asBoolean()) {
+ request.setMessageStructure("json");
+ }
+
+ final String arn = context.getProperty(ARN).evaluateAttributeExpressions(flowFile).getValue();
+ final String arnType = context.getProperty(ARN_TYPE).getValue();
+ if (arnType.equalsIgnoreCase(ARN_TYPE_TOPIC.getValue())) {
+ request.setTopicArn(arn);
+ } else {
+ request.setTargetArn(arn);
+ }
+
+ final String subject = context.getProperty(SUBJECT).evaluateAttributeExpressions(flowFile).getValue();
+ if (subject != null) {
+ request.setSubject(subject);
+ }
+
+ for (final Map.Entry<PropertyDescriptor, String> entry : context.getProperties().entrySet()) {
+ if (entry.getKey().isDynamic() && !isEmpty(entry.getValue())) {
+ final MessageAttributeValue value = new MessageAttributeValue();
+ value.setStringValue(context.getProperty(entry.getKey()).evaluateAttributeExpressions(flowFile).getValue());
+ value.setDataType("String");
+ request.addMessageAttributesEntry(entry.getKey().getName(), value);
+ }
+ }
+
+ try {
+ client.publish(request);
+ session.transfer(flowFile, REL_SUCCESS);
+ session.getProvenanceReporter().send(flowFile, arn);
+ getLogger().info("Successfully published notification for {}", new Object[]{flowFile});
+ } catch (final Exception e) {
+ getLogger().error("Failed to publish Amazon SNS message for {} due to {}", new Object[]{flowFile, e});
+ session.transfer(flowFile, REL_FAILURE);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/sqs/AbstractSQSProcessor.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/sqs/AbstractSQSProcessor.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/sqs/AbstractSQSProcessor.java
new file mode 100644
index 0000000..3cee02d
--- /dev/null
+++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/sqs/AbstractSQSProcessor.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.aws.sqs;
+
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.processors.aws.AbstractAWSProcessor;
+
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.services.sqs.AmazonSQSClient;
+
+public abstract class AbstractSQSProcessor extends AbstractAWSProcessor<AmazonSQSClient> {
+
+ public static final PropertyDescriptor BATCH_SIZE = new PropertyDescriptor.Builder()
+ .name("Batch Size")
+ .description("The maximum number of messages to send in a single network request")
+ .required(true)
+ .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR)
+ .defaultValue("25")
+ .build();
+
+ public static final PropertyDescriptor QUEUE_URL = new PropertyDescriptor.Builder()
+ .name("Queue URL")
+ .description("The URL of the queue to act upon")
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .expressionLanguageSupported(true)
+ .required(true)
+ .build();
+
+ @Override
+ protected AmazonSQSClient createClient(final ProcessContext context, final AWSCredentials credentials, final ClientConfiguration config) {
+ return new AmazonSQSClient(credentials, config);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/sqs/DeleteSQS.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/sqs/DeleteSQS.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/sqs/DeleteSQS.java
new file mode 100644
index 0000000..65e020d
--- /dev/null
+++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/sqs/DeleteSQS.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.aws.sqs;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.SeeAlso;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.util.StandardValidators;
+
+import com.amazonaws.services.sqs.AmazonSQSClient;
+import com.amazonaws.services.sqs.model.DeleteMessageBatchRequest;
+import com.amazonaws.services.sqs.model.DeleteMessageBatchRequestEntry;
+
+@SupportsBatching
+@SeeAlso({GetSQS.class, PutSQS.class})
+@Tags({"Amazon", "AWS", "SQS", "Queue", "Delete"})
+@CapabilityDescription("Deletes a message from an Amazon Simple Queuing Service Queue")
+public class DeleteSQS extends AbstractSQSProcessor {
+
+ public static final PropertyDescriptor RECEIPT_HANDLE = new PropertyDescriptor.Builder()
+ .name("Receipt Handle")
+ .description("The identifier that specifies the receipt of the message")
+ .expressionLanguageSupported(true)
+ .required(true)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .defaultValue("${sqs.receipt.handle}")
+ .build();
+
+ public static final List<PropertyDescriptor> properties = Collections.unmodifiableList(
+ Arrays.asList(ACCESS_KEY, SECRET_KEY, REGION, QUEUE_URL, TIMEOUT));
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return properties;
+ }
+
+ @Override
+ public void onTrigger(final ProcessContext context, final ProcessSession session) {
+ List<FlowFile> flowFiles = session.get(1);
+ if (flowFiles.isEmpty()) {
+ return;
+ }
+
+ final FlowFile firstFlowFile = flowFiles.get(0);
+ final String queueUrl = context.getProperty(QUEUE_URL).evaluateAttributeExpressions(firstFlowFile).getValue();
+
+ final AmazonSQSClient client = getClient();
+ final DeleteMessageBatchRequest request = new DeleteMessageBatchRequest();
+ request.setQueueUrl(queueUrl);
+
+ final List<DeleteMessageBatchRequestEntry> entries = new ArrayList<>(flowFiles.size());
+
+ for (final FlowFile flowFile : flowFiles) {
+ final DeleteMessageBatchRequestEntry entry = new DeleteMessageBatchRequestEntry();
+ entry.setReceiptHandle(context.getProperty(RECEIPT_HANDLE).evaluateAttributeExpressions(flowFile).getValue());
+ entries.add(entry);
+ }
+
+ request.setEntries(entries);
+
+ try {
+ client.deleteMessageBatch(request);
+ getLogger().info("Successfully deleted {} objects from SQS", new Object[]{flowFiles.size()});
+ session.transfer(flowFiles, REL_SUCCESS);
+ } catch (final Exception e) {
+ getLogger().error("Failed to delete {} objects from SQS due to {}", new Object[]{flowFiles.size(), e});
+ session.transfer(flowFiles, REL_FAILURE);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/sqs/GetSQS.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/sqs/GetSQS.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/sqs/GetSQS.java
new file mode 100644
index 0000000..7c2dd2d
--- /dev/null
+++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/sqs/GetSQS.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.aws.sqs;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.SeeAlso;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.io.OutputStreamCallback;
+import org.apache.nifi.processor.util.StandardValidators;
+
+import com.amazonaws.services.sqs.AmazonSQSClient;
+import com.amazonaws.services.sqs.model.DeleteMessageBatchRequest;
+import com.amazonaws.services.sqs.model.DeleteMessageBatchRequestEntry;
+import com.amazonaws.services.sqs.model.Message;
+import com.amazonaws.services.sqs.model.MessageAttributeValue;
+import com.amazonaws.services.sqs.model.ReceiveMessageRequest;
+import com.amazonaws.services.sqs.model.ReceiveMessageResult;
+
+@SupportsBatching
+@Tags({"Amazon", "AWS", "SQS", "Queue", "Get", "Fetch", "Poll"})
+@SeeAlso({PutSQS.class, DeleteSQS.class})
+@CapabilityDescription("Fetches messages from an Amazon Simple Queuing Service Queue")
+@WritesAttributes({
+ @WritesAttribute(attribute = "hash.value", description = "The MD5 sum of the message"),
+ @WritesAttribute(attribute = "hash.algorithm", description = "MD5"),
+ @WritesAttribute(attribute = "sqs.message.id", description = "The unique identifier of the SQS message"),
+ @WritesAttribute(attribute = "sqs.receipt.handle", description = "The SQS Receipt Handle that is to be used to delete the message from the queue")
+})
+public class GetSQS extends AbstractSQSProcessor {
+
+ public static final PropertyDescriptor CHARSET = new PropertyDescriptor.Builder()
+ .name("Character Set")
+ .description("The Character Set that should be used to encode the textual content of the SQS message")
+ .required(true)
+ .defaultValue("UTF-8")
+ .allowableValues(Charset.availableCharsets().keySet().toArray(new String[0]))
+ .build();
+
+ public static final PropertyDescriptor AUTO_DELETE = new PropertyDescriptor.Builder()
+ .name("Auto Delete Messages")
+ .description("Specifies whether the messages should be automatically deleted by the processors once they have been received.")
+ .required(true)
+ .allowableValues("true", "false")
+ .defaultValue("true")
+ .build();
+
+ public static final PropertyDescriptor VISIBILITY_TIMEOUT = new PropertyDescriptor.Builder()
+ .name("Visibility Timeout")
+ .description("The amount of time after a message is received but not deleted that the message is hidden from other consumers")
+ .expressionLanguageSupported(false)
+ .required(true)
+ .defaultValue("15 mins")
+ .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR)
+ .build();
+
+ public static final PropertyDescriptor BATCH_SIZE = new PropertyDescriptor.Builder()
+ .name("Batch Size")
+ .description("The maximum number of messages to send in a single network request")
+ .required(true)
+ .addValidator(StandardValidators.createLongValidator(1L, 10L, true))
+ .defaultValue("10")
+ .build();
+
+ public static final PropertyDescriptor STATIC_QUEUE_URL = new PropertyDescriptor.Builder()
+ .fromPropertyDescriptor(QUEUE_URL)
+ .expressionLanguageSupported(false)
+ .build();
+
+ public static final List<PropertyDescriptor> properties = Collections.unmodifiableList(
+ Arrays.asList(STATIC_QUEUE_URL, AUTO_DELETE, ACCESS_KEY, SECRET_KEY, CREDENTAILS_FILE, REGION, BATCH_SIZE, TIMEOUT, CHARSET, VISIBILITY_TIMEOUT));
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return properties;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return Collections.singleton(REL_SUCCESS);
+ }
+
+ @Override
+ public void onTrigger(final ProcessContext context, final ProcessSession session) {
+ final String queueUrl = context.getProperty(STATIC_QUEUE_URL).getValue();
+
+ final AmazonSQSClient client = getClient();
+
+ final ReceiveMessageRequest request = new ReceiveMessageRequest();
+ request.setAttributeNames(Collections.singleton("All"));
+ request.setMaxNumberOfMessages(context.getProperty(BATCH_SIZE).asInteger());
+ request.setVisibilityTimeout(context.getProperty(VISIBILITY_TIMEOUT).asTimePeriod(TimeUnit.SECONDS).intValue());
+ request.setQueueUrl(queueUrl);
+
+ final Charset charset = Charset.forName(context.getProperty(CHARSET).getValue());
+
+ final ReceiveMessageResult result;
+ try {
+ result = client.receiveMessage(request);
+ } catch (final Exception e) {
+ getLogger().error("Failed to receive messages from Amazon SQS due to {}", new Object[]{e});
+ context.yield();
+ return;
+ }
+
+ final List<Message> messages = result.getMessages();
+ if (messages.isEmpty()) {
+ context.yield();
+ return;
+ }
+
+ final boolean autoDelete = context.getProperty(AUTO_DELETE).asBoolean();
+
+ for (final Message message : messages) {
+ FlowFile flowFile = session.create();
+
+ final Map<String, String> attributes = new HashMap<>();
+ for (final Map.Entry<String, String> entry : message.getAttributes().entrySet()) {
+ attributes.put("sqs." + entry.getKey(), entry.getValue());
+ }
+
+ for (final Map.Entry<String, MessageAttributeValue> entry : message.getMessageAttributes().entrySet()) {
+ attributes.put("sqs." + entry.getKey(), entry.getValue().getStringValue());
+ }
+
+ attributes.put("hash.value", message.getMD5OfBody());
+ attributes.put("hash.algorithm", "md5");
+ attributes.put("sqs.message.id", message.getMessageId());
+ attributes.put("sqs.receipt.handle", message.getReceiptHandle());
+
+ flowFile = session.putAllAttributes(flowFile, attributes);
+ flowFile = session.write(flowFile, new OutputStreamCallback() {
+ @Override
+ public void process(final OutputStream out) throws IOException {
+ out.write(message.getBody().getBytes(charset));
+ }
+ });
+
+ session.transfer(flowFile, REL_SUCCESS);
+ session.getProvenanceReporter().receive(flowFile, queueUrl);
+
+ getLogger().info("Successfully received {} from Amazon SQS", new Object[]{flowFile});
+ }
+
+ if (autoDelete) {
+ // If we want to auto-delete messages, we must fist commit the session to ensure that the data
+ // is persisted in NiFi's repositories.
+ session.commit();
+
+ final DeleteMessageBatchRequest deleteRequest = new DeleteMessageBatchRequest();
+ deleteRequest.setQueueUrl(queueUrl);
+ final List<DeleteMessageBatchRequestEntry> deleteRequestEntries = new ArrayList<>();
+ for (final Message message : messages) {
+ final DeleteMessageBatchRequestEntry entry = new DeleteMessageBatchRequestEntry();
+ entry.setId(message.getMessageId());
+ entry.setReceiptHandle(message.getReceiptHandle());
+ deleteRequestEntries.add(entry);
+ }
+
+ deleteRequest.setEntries(deleteRequestEntries);
+
+ try {
+ client.deleteMessageBatch(deleteRequest);
+ } catch (final Exception e) {
+ getLogger().error("Received {} messages from Amazon SQS but failed to delete the messages; these messages"
+ + " may be duplicated. Reason for deletion failure: {}", new Object[]{messages.size(), e});
+ }
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/sqs/PutSQS.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/sqs/PutSQS.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/sqs/PutSQS.java
new file mode 100644
index 0000000..3961f32
--- /dev/null
+++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/sqs/PutSQS.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.aws.sqs;
+
+import java.io.ByteArrayOutputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.nifi.annotation.behavior.DynamicProperty;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.SeeAlso;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.util.StandardValidators;
+
+import com.amazonaws.services.sqs.AmazonSQSClient;
+import com.amazonaws.services.sqs.model.MessageAttributeValue;
+import com.amazonaws.services.sqs.model.SendMessageBatchRequest;
+import com.amazonaws.services.sqs.model.SendMessageBatchRequestEntry;
+
+@SupportsBatching
+@Tags({"Amazon", "AWS", "SQS", "Queue", "Put", "Publish"})
+@SeeAlso({GetSQS.class, DeleteSQS.class})
+@CapabilityDescription("Publishes a message to an Amazon Simple Queuing Service Queue")
+@DynamicProperty(name = "The name of a Message Attribute to add to the message", value = "The value of the Message Attribute",
+ description = "Allows the user to add key/value pairs as Message Attributes by adding a property whose name will become the name of "
+ + "the Message Attribute and value will become the value of the Message Attribute", supportsExpressionLanguage = true)
+public class PutSQS extends AbstractSQSProcessor {
+
+ public static final PropertyDescriptor DELAY = new PropertyDescriptor.Builder()
+ .name("Delay")
+ .description("The amount of time to delay the message before it becomes available to consumers")
+ .required(true)
+ .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR)
+ .defaultValue("0 secs")
+ .build();
+
+ public static final List<PropertyDescriptor> properties = Collections.unmodifiableList(
+ Arrays.asList(QUEUE_URL, ACCESS_KEY, SECRET_KEY, CREDENTAILS_FILE, REGION, DELAY, TIMEOUT));
+
+ private volatile List<PropertyDescriptor> userDefinedProperties = Collections.emptyList();
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return properties;
+ }
+
+ @Override
+ protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) {
+ return new PropertyDescriptor.Builder()
+ .name(propertyDescriptorName)
+ .expressionLanguageSupported(true)
+ .required(false)
+ .dynamic(true)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+ }
+
+ @OnScheduled
+ public void setup(final ProcessContext context) {
+ userDefinedProperties = new ArrayList<>();
+ for (final PropertyDescriptor descriptor : context.getProperties().keySet()) {
+ if (descriptor.isDynamic()) {
+ userDefinedProperties.add(descriptor);
+ }
+ }
+ }
+
+ @Override
+ public void onTrigger(final ProcessContext context, final ProcessSession session) {
+ FlowFile flowFile = session.get();
+ if (flowFile == null) {
+ return;
+ }
+
+ final long startNanos = System.nanoTime();
+ final AmazonSQSClient client = getClient();
+ final SendMessageBatchRequest request = new SendMessageBatchRequest();
+ final String queueUrl = context.getProperty(QUEUE_URL).evaluateAttributeExpressions(flowFile).getValue();
+ request.setQueueUrl(queueUrl);
+
+ final Set<SendMessageBatchRequestEntry> entries = new HashSet<>();
+
+ final SendMessageBatchRequestEntry entry = new SendMessageBatchRequestEntry();
+ entry.setId(flowFile.getAttribute("uuid"));
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ session.exportTo(flowFile, baos);
+ final String flowFileContent = baos.toString();
+ entry.setMessageBody(flowFileContent);
+
+ final Map<String, MessageAttributeValue> messageAttributes = new HashMap<>();
+
+ for (final PropertyDescriptor descriptor : userDefinedProperties) {
+ final MessageAttributeValue mav = new MessageAttributeValue();
+ mav.setDataType("String");
+ mav.setStringValue(context.getProperty(descriptor).evaluateAttributeExpressions(flowFile).getValue());
+ messageAttributes.put(descriptor.getName(), mav);
+ }
+
+ entry.setMessageAttributes(messageAttributes);
+ entry.setDelaySeconds(context.getProperty(DELAY).asTimePeriod(TimeUnit.SECONDS).intValue());
+ entries.add(entry);
+
+ request.setEntries(entries);
+
+ try {
+ client.sendMessageBatch(request);
+ } catch (final Exception e) {
+ getLogger().error("Failed to send messages to Amazon SQS due to {}; routing to failure", new Object[]{e});
+ session.transfer(flowFile, REL_FAILURE);
+ return;
+ }
+
+ getLogger().info("Successfully published message to Amazon SQS for {}", new Object[]{flowFile});
+ session.transfer(flowFile, REL_SUCCESS);
+ final long transmissionMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
+ session.getProvenanceReporter().send(flowFile, queueUrl, transmissionMillis);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
new file mode 100644
index 0000000..4f2405c
--- /dev/null
+++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+org.apache.nifi.processors.aws.s3.FetchS3Object
+org.apache.nifi.processors.aws.s3.PutS3Object
+org.apache.nifi.processors.aws.sns.PutSNS
+org.apache.nifi.processors.aws.sqs.GetSQS
+org.apache.nifi.processors.aws.sqs.PutSQS
+org.apache.nifi.processors.aws.sqs.DeleteSQS
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestFetchS3Object.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestFetchS3Object.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestFetchS3Object.java
new file mode 100644
index 0000000..0321514
--- /dev/null
+++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestFetchS3Object.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.aws.s3;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.Ignore;
+import org.junit.Test;
+
+@Ignore("For local testing only - interacts with S3 so the credentials file must be configured and all necessary buckets created")
+public class TestFetchS3Object {
+
+ private final String CREDENTIALS_FILE = System.getProperty("user.home") + "/aws-credentials.properties";
+
+ @Test
+ public void testGet() throws IOException {
+ final TestRunner runner = TestRunners.newTestRunner(new FetchS3Object());
+ runner.setProperty(FetchS3Object.BUCKET, "anonymous-test-bucket-00000000");
+ runner.setProperty(FetchS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
+ runner.setProperty(FetchS3Object.KEY, "folder/1.txt");
+
+ final Map<String, String> attrs = new HashMap<>();
+ attrs.put("start", "0");
+
+ runner.enqueue(new byte[0], attrs);
+ runner.run(1);
+
+ runner.assertAllFlowFilesTransferred(FetchS3Object.REL_SUCCESS, 1);
+ final List<MockFlowFile> ffs = runner.getFlowFilesForRelationship(FetchS3Object.REL_SUCCESS);
+ final MockFlowFile out = ffs.iterator().next();
+
+ final byte[] expectedBytes = Files.readAllBytes(Paths.get("src/test/resources/hello.txt"));
+ out.assertContentEquals(new String(expectedBytes));
+ for (final Map.Entry<String, String> entry : out.getAttributes().entrySet()) {
+ System.out.println(entry.getKey() + " : " + entry.getValue());
+ }
+ }
+
+}
[15/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/invalid-processor.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/invalid-processor.png b/nifi-docs/src/main/asciidoc/images/invalid-processor.png
new file mode 100644
index 0000000..37117b4
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/invalid-processor.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/lineage-flowfile.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/lineage-flowfile.png b/nifi-docs/src/main/asciidoc/images/lineage-flowfile.png
new file mode 100644
index 0000000..29bdc5b
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/lineage-flowfile.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/lineage-graph-annotated.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/lineage-graph-annotated.png b/nifi-docs/src/main/asciidoc/images/lineage-graph-annotated.png
new file mode 100644
index 0000000..31bad42
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/lineage-graph-annotated.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/ncm.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/ncm.png b/nifi-docs/src/main/asciidoc/images/ncm.png
new file mode 100644
index 0000000..a63e313
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/ncm.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/new-flow.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/new-flow.png b/nifi-docs/src/main/asciidoc/images/new-flow.png
new file mode 100644
index 0000000..26a6b8f
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/new-flow.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/nifi-arch-cluster.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/nifi-arch-cluster.png b/nifi-docs/src/main/asciidoc/images/nifi-arch-cluster.png
new file mode 100644
index 0000000..b9a067f
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/nifi-arch-cluster.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/nifi-arch.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/nifi-arch.png b/nifi-docs/src/main/asciidoc/images/nifi-arch.png
new file mode 100644
index 0000000..dad81fa
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/nifi-arch.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/nifi-navigation.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/nifi-navigation.png b/nifi-docs/src/main/asciidoc/images/nifi-navigation.png
new file mode 100644
index 0000000..48c0f24
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/nifi-navigation.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/nifi-toolbar-components.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/nifi-toolbar-components.png b/nifi-docs/src/main/asciidoc/images/nifi-toolbar-components.png
new file mode 100644
index 0000000..9edaa4c
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/nifi-toolbar-components.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/nifi_first_launch_screenshot.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/nifi_first_launch_screenshot.png b/nifi-docs/src/main/asciidoc/images/nifi_first_launch_screenshot.png
new file mode 100644
index 0000000..7735f7e
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/nifi_first_launch_screenshot.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/parent-found.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/parent-found.png b/nifi-docs/src/main/asciidoc/images/parent-found.png
new file mode 100644
index 0000000..f27244d
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/parent-found.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/process-group-anatomy.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/process-group-anatomy.png b/nifi-docs/src/main/asciidoc/images/process-group-anatomy.png
new file mode 100644
index 0000000..dd09218
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/process-group-anatomy.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/processor-anatomy.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/processor-anatomy.png b/nifi-docs/src/main/asciidoc/images/processor-anatomy.png
new file mode 100644
index 0000000..db44d6a
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/processor-anatomy.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/processor-connection-bubble.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/processor-connection-bubble.png b/nifi-docs/src/main/asciidoc/images/processor-connection-bubble.png
new file mode 100644
index 0000000..82a161b
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/processor-connection-bubble.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/properties-tab.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/properties-tab.png b/nifi-docs/src/main/asciidoc/images/properties-tab.png
new file mode 100644
index 0000000..0e60fb9
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/properties-tab.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/provenance-annotated.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/provenance-annotated.png b/nifi-docs/src/main/asciidoc/images/provenance-annotated.png
new file mode 100644
index 0000000..f00cc4c
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/provenance-annotated.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/provenance-table.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/provenance-table.png b/nifi-docs/src/main/asciidoc/images/provenance-table.png
new file mode 100644
index 0000000..a0ca075
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/provenance-table.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/remote-group-anatomy.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/remote-group-anatomy.png b/nifi-docs/src/main/asciidoc/images/remote-group-anatomy.png
new file mode 100644
index 0000000..3403aeb
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/remote-group-anatomy.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/remote-group-ports-dialog.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/remote-group-ports-dialog.png b/nifi-docs/src/main/asciidoc/images/remote-group-ports-dialog.png
new file mode 100644
index 0000000..7b13b00
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/remote-group-ports-dialog.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/remote-port-connection-status.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/remote-port-connection-status.png b/nifi-docs/src/main/asciidoc/images/remote-port-connection-status.png
new file mode 100644
index 0000000..2e7cecf
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/remote-port-connection-status.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/reporting-tasks-edit-buttons.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/reporting-tasks-edit-buttons.png b/nifi-docs/src/main/asciidoc/images/reporting-tasks-edit-buttons.png
new file mode 100644
index 0000000..f115bd4
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/reporting-tasks-edit-buttons.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/reporting-tasks-edit-buttons2.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/reporting-tasks-edit-buttons2.png b/nifi-docs/src/main/asciidoc/images/reporting-tasks-edit-buttons2.png
new file mode 100644
index 0000000..70a9d50
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/reporting-tasks-edit-buttons2.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/reporting-tasks-tab.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/reporting-tasks-tab.png b/nifi-docs/src/main/asciidoc/images/reporting-tasks-tab.png
new file mode 100644
index 0000000..26e8324
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/reporting-tasks-tab.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/scheduling-tab.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/scheduling-tab.png b/nifi-docs/src/main/asciidoc/images/scheduling-tab.png
new file mode 100644
index 0000000..20ca0f5
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/scheduling-tab.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/search-events.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/search-events.png b/nifi-docs/src/main/asciidoc/images/search-events.png
new file mode 100644
index 0000000..31c153b
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/search-events.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/search-receive-event-abc.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/search-receive-event-abc.png b/nifi-docs/src/main/asciidoc/images/search-receive-event-abc.png
new file mode 100644
index 0000000..f9312cd
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/search-receive-event-abc.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/settings-general-tab.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/settings-general-tab.png b/nifi-docs/src/main/asciidoc/images/settings-general-tab.png
new file mode 100644
index 0000000..2260581
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/settings-general-tab.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/settings-tab.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/settings-tab.png b/nifi-docs/src/main/asciidoc/images/settings-tab.png
new file mode 100644
index 0000000..77d6bd7
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/settings-tab.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/simple-flow.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/simple-flow.png b/nifi-docs/src/main/asciidoc/images/simple-flow.png
new file mode 100644
index 0000000..85203a8
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/simple-flow.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/stats-history.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/stats-history.png b/nifi-docs/src/main/asciidoc/images/stats-history.png
new file mode 100644
index 0000000..89ce235
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/stats-history.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/status-bar.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/status-bar.png b/nifi-docs/src/main/asciidoc/images/status-bar.png
new file mode 100644
index 0000000..879be06
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/status-bar.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/summary-annotated.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/summary-annotated.png b/nifi-docs/src/main/asciidoc/images/summary-annotated.png
new file mode 100644
index 0000000..30c5b10
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/summary-annotated.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/summary-table.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/summary-table.png b/nifi-docs/src/main/asciidoc/images/summary-table.png
new file mode 100644
index 0000000..609ec7c
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/summary-table.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/images/valid-processor.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/valid-processor.png b/nifi-docs/src/main/asciidoc/images/valid-processor.png
new file mode 100644
index 0000000..d6102b7
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/valid-processor.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/overview.adoc
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/overview.adoc b/nifi-docs/src/main/asciidoc/overview.adoc
new file mode 100644
index 0000000..af5a907
--- /dev/null
+++ b/nifi-docs/src/main/asciidoc/overview.adoc
@@ -0,0 +1,296 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+Apache NiFi Overview
+====================
+Apache NiFi Team <de...@nifi.apache.org>
+:homepage: http://nifi.apache.org
+
+What is Apache NiFi?
+--------------------
+Put simply NiFi was built to automate the flow of data between systems. While
+the term 'dataflow' is used in a variety of contexts, we'll use it here
+to mean the automated and managed flow of information between systems. This
+problem space has been around ever since enterprises had more than one system,
+where some of the systems created data and some of the systems consumed data.
+The problems and solution patterns that emerged have been discussed and
+articulated extensively. A comprehensive and readily consumed form is found in
+the _Enterprise Integration Patterns_ <<eip>>.
+
+Some of the high-level challenges of dataflow include:
+
+Systems fail::
+Networks fail, disks fail, software crashes, people make mistakes.
+
+Data access exceeds capacity to consume::
+Sometimes a given data source can outpace some part of the processing or delivery chain - it only takes one weak-link to have an issue.
+
+Boundary conditions are mere suggestions::
+You will invariably get data that is too big, too small, too fast, too slow, corrupt, wrong, or in the wrong format.
+
+What is noise one day becomes signal the next::
+Priorities of an organization change - rapidly. Enabling new flows and changing existing ones must be fast.
+
+Systems evolve at different rates::
+The protocols and formats used by a given system can change anytime and often irrespective of the systems around them. Dataflow exists to connect what is essentially a massively distributed system of components that are loosely or not-at-all designed to work together.
+
+Compliance and security::
+Laws, regulations, and policies change. Business to business agreements change. System to system and system to user interactions must be secure, trusted, accountable.
+
+Continuous improvement occurs in production::
+It is often not possible to come even close to replicating production environments in the lab.
+
+Over the years dataflow has been one of those necessary evils in an
+architecture. Now though there are a number of active and rapidly evolving
+movements making dataflow a lot more interesting and a lot more vital to the
+success of a given enterprise. These include things like; Service Oriented
+Architecture <<soa>>, the rise of the API <<api>><<api2>>, Internet of Things <<iot>>,
+and Big Data <<bigdata>>. In addition, the level of rigor necessary for
+compliance, privacy, and security is constantly on the rise. Even still with
+all of these new concepts coming about, the patterns and needs of dataflow are
+still largely the same. The primary differences then are the scope of
+complexity, the rate of change necessary to adapt, and that at scale
+the edge case becomes common occurrence. NiFi is built to help tackle these
+modern dataflow challenges.
+
+The core concepts of NiFi
+-------------------------
+
+NiFi's fundamental design concepts closely relate to the main ideas of Flow Based
+Programming <<fbp>>. Here are some of
+the main NiFi concepts and how they map to FBP:
+[grid="rows"]
+[options="header",cols="3,3,10"]
+|===========================
+| NiFi Term | FBP Term| Description
+
+| FlowFile | Information Packet |
+A FlowFile represents each object moving through the system and for each one, NiFi
+keeps track of a map of key/value pair attribute strings and its associated
+content of zero or more bytes.
+
+| FlowFile Processor | Black Box |
+Processors actually perform the work. In <<eip>> terms a processor is
+doing some combination of data Routing, Transformation, or Mediation between
+systems. Processors have access to attributes of a given FlowFile and its
+content stream. Processors can operate on zero or more FlowFiles in a given unit of work
+and either commit that work or rollback.
+
+| Connection | Bounded Buffer |
+Connections provide the actual linkage between processors. These act as queues
+and allow various processes to interact at differing rates. These queues then
+can be prioritized dynamically and can have upper bounds on load, which enable
+back pressure.
+
+| Flow Controller | Scheduler |
+The Flow Controller maintains the knowledge of how processes actually connect
+and manages the threads and allocations thereof which all processes use. The
+Flow Controller acts as the broker facilitating the exchange of FlowFiles
+between processors.
+
+| Process Group | subnet |
+A Process Group is a specific set of processes and their connections, which can
+receive data via input ports and send data out via output ports. In
+this manner process groups allow creation of entirely new components simply by
+composition of other components.
+
+|===========================
+
+This design model, also similar to <<seda>>, provides many beneficial consequences that help NiFi
+to be a very effective platform for building powerful and scalable dataflows.
+A few of these benefits include:
+
+* Lends well to visual creation and management of directed graphs of processors
+* Is inherently asynchronous which allows for very high throughput and natural buffering even as processing and flow rates fluctuate
+* Provides a highly concurrent model without a developer having to worry about the typical complexities of concurrency
+* Promotes the development of cohesive and loosely coupled components which can then be reused in other contexts and promotes testable units
+* The resource constrained connections make critical functions such as back-pressure and pressure release very natural and intuitive
+* Error handling becomes as natural as the happy-path rather than a coarse grained catch-all
+* The points at which data enters and exits the system as well as how it flows through are well understood and easily tracked
+
+NiFi Architecture
+-----------------
+image::nifi-arch.png["NiFi Architecture Diagram"]
+
+NiFi executes within a JVM living within a host operating system. The primary
+components of NiFi then living within the JVM are as follows:
+
+Web Server::
+The purpose of the web server is to host NiFi's HTTP-based command and control API.
+
+Flow Controller::
+The flow controller is the brains of the operation. It provides threads for extensions to run on and manages their schedule of when they'll receive resources to execute.
+
+Extensions::
+There are various types of extensions for NiFi which will be described in other documents. But the key point here is that extensions operate/execute within the JVM.
+
+FlowFile Repository::
+The FlowFile Repository is where NiFi keeps track of the state of what it knows about a given FlowFile that is presently active in the flow. The implementation of the repository is pluggable. The default approach is a persistent Write-Ahead Log that lives on a specified disk partition.
+
+Content Repository::
+The Content Repository is where the actual content bytes of a given FlowFile live. The implementation of the repository is pluggable. The default approach is a fairly simple mechanism, which stores blocks of data in the file system. More than one file system storage location can be specified so as to get different physical partitions engaged to reduce contention on any single volume.
+
+Provenance Repository::
+The Provenance Repository is where all provenance event data is stored. The repository construct is pluggable with the default implementation being to use one or more physical disk volumes. Within each location event data is indexed and searchable.
+
+NiFi is also able to operate within a cluster.
+
+image::nifi-arch-cluster.png["NiFi Cluster Architecture Diagram"]
+
+A NiFi cluster is comprised of one or more 'NiFi Nodes' (Node) controlled
+by a single NiFi Cluster Manager (NCM). The design of clustering is a simple
+master/slave model where the NCM is the master and the Nodes are the slaves.
+The NCM's reason for existence is to keep track of which Nodes are in the cluster,
+their status, and to replicate requests to modify or observe the
+flow. Fundamentally, then, the NCM keeps the state of the cluster consistent.
+While the model is that of master and slave, if the master dies the Nodes are all
+instructed to continue operating as they were to ensure the data flow remains live.
+The absence of the NCM simply means new nodes cannot join the cluster and cluster flow changes
+cannot occur until the NCM is restored.
+
+Performance Expectations and Characteristics of NiFi
+----------------------------------------------------
+NiFi is designed to fully leverage the capabilities of the underlying host system
+it is operating on. This maximization of resources is particularly strong with
+regard to CPU and disk. Many more details will
+be provided on best practices and configuration tips in the Administration Guide.
+
+For IO::
+The throughput or latency
+one can expect to see will vary greatly on how the system is configured. Given
+that there are pluggable approaches to most of the major NiFi subsystems the
+performance will depend on the implementation. But, for something concrete and broadly
+applicable, let's consider the out-of-the-box default implementations that are used.
+These are all persistent with guaranteed delivery and do so using local disk. So
+being conservative, assume roughly 50 MB/s read/write rate on modest disks or RAID volumes
+within a typical server. NiFi for a large class of dataflows then should be able to
+efficiently reach 100 or more MB/s of throughput. That is because linear growth
+is expected for each physical partition and content repository added to NiFi. This will
+bottleneck at some point on the FlowFile repository and provenance repository.
+We plan to provide a benchmarking/performance test template to
+include in the build, which will allow users to easily test their system and
+to identify where bottlenecks are and at which point they might become a factor. It
+should also make it easy for system administrators to make changes and to verify the impact.
+
+For CPU::
+The Flow Controller acts as the engine dictating when a particular processor will be
+given a thread to execute. Processors should be written to return the thread
+as soon as they're done executing their task. The Flow Controller can be given a
+configuration value indicating how many threads there should be for the various
+thread pools it maintains. The ideal number of threads to use will depend on the
+resources of the host system in terms of numbers of cores, whether that system is
+running other services as well, and the nature of the processing in the flow. For
+typical IO heavy flows though it would be quite reasonable to set many dozens of threads
+to be available if not more.
+
+For RAM::
+NiFi lives within the JVM and is thus generally limited to the memory space it
+is afforded by the JVM. Garbage collection of the JVM becomes a very important
+factor to both restricting the total practical size the heap can be as well as
+how well the application will run over time.
+
+High Level Overview of Key NiFi Features
+----------------------------------------
+Guaranteed Delivery::
+A core philosophy of NiFi has been that even at very high scale, guaranteed delivery
+is a must. This is achieved through effective use of a purpose-built persistent
+write-ahead log and content repository. Together they are designed in such a way
+as to allow for very high transaction rates, effective load-spreading, copy-on-write,
+and play to the strengths of traditional disk read/writes.
+
+Data Buffering w/ Back Pressure and Pressure Release::
+NiFi supports buffering of all queued data as well as the ability to
+provide back pressure as those queues reach specified limits or to age off data
+as it reaches a specified age (its value has perished).
+
+Prioritized Queuing::
+NiFi allows the setting of one or more prioritization schemes for how data is
+retrieved from a queue. The default is oldest first, but there are times when
+data should be pulled newest first, largest first, or some other custom scheme.
+
+Flow Specific QoS (latency v throughput, loss tolerance, etc.)::
+There are points of a dataflow where the data is absolutely critical and it is
+loss intolerant. There are also times when it must be processed and delivered within
+seconds to be of any value. NiFi enables the fine-grained flow specific configuration
+of these concerns.
+
+Data Provenance::
+NiFi automatically records, indexes, and makes available provenance data as
+objects flow through the system even across fan-in, fan-out, transformations, and
+more. This information becomes extremely critical in supporting compliance,
+troubleshooting, optimization, and other scenarios.
+
+Recovery / Recording a rolling buffer of fine-grained history::
+NiFi's content repository is designed to act as a rolling buffer of history. Data
+is removed only as it ages off the content repository or as space is needed. This
+combined with the data provenance capability makes for an incredibly useful basis
+to enable click-to-content, download of content, and replay, all at a specific
+point in an object's lifecycle which can even span generations.
+
+Visual Command and Control::
+Dataflows can become quite complex. Being able to visualize those flows and express
+them visually can help greatly to reduce that complexity and to identify areas that
+need to be simplified. NiFi enables not only the visual establishment of dataflows but
+it does so in real-time. Rather than being 'design and deploy' it is much more like
+molding clay. If you make a change to the dataflow that change immediately takes effect. Changes
+are fine-grained and isolated to the affected components. You don't need to stop an entire
+flow or set of flows just to make some specific modification.
+
+Flow Templates::
+Dataflows tend to be highly pattern oriented and while there are often many different
+ways to solve a problem, it helps greatly to be able to share those best practices. Templates
+allow subject matter experts to build and publish their flow designs and for others to benefit
+and collaborate on them.
+
+Security::
+ System to system;;
+ A dataflow is only as good as it is secure. NiFi at every point in a dataflow offers secure
+ exchange through the use of protocols with encryption such as 2-way SSL. In addition
+ NiFi enables the flow to encrypt and decrypt content and use shared-keys or other mechanisms on
+ either side of the sender/recipient equation.
+ User to system;;
+ NiFi enables 2-Way SSL authentication and provides pluggable authorization so that it can properly control
+ a user's access and at particular levels (read-only, dataflow manager, admin). If a user enters a
+ sensitive property like a password into the flow, it is immediately encrypted server side and never again exposed
+ on the client side even in its encrypted form.
+
+Designed for Extension::
+ NiFi is at its core built for extension and as such it is a platform on which dataflow processes can execute and interact in a predictable and repeatable manner.
+ Points of extension;;
+ Processors, Controller Services, Reporting Tasks, Prioritizers, Customer User Interfaces
+ Classloader Isolation;;
+ For any component-based system, dependency nightmares can quickly occur. NiFi addresses this by providing a custom class loader model,
+ ensuring that each extension bundle is exposed to a very limited set of dependencies. As a result, extensions can be built with little concern for whether
+ they might conflict with another extension. The concept of these extension bundles is called 'NiFi Archives' and will be discussed in greater detail
+ in the developer's guide.
+Clustering (scale-out)::
+ NiFi is designed to scale-out through the use of clustering many nodes together as described above. If a single node is provisioned and configured
+ to handle hundreds of MB/s then a modest cluster could be configured to handle GB/s. This then brings about interesting challenges of load balancing
+ and fail-over between NiFi and the systems from which it gets data. Use of asynchronous queuing based protocols like messaging services, Kafka, etc., can
+ help. Use of NiFi's 'site-to-site' feature is also very effective as it is a protocol that allows NiFi and a client (could be another NiFi cluster) to talk to each other, share information
+ about loading, and to exchange data on specific authorized ports.
+
+References
+----------
+[bibliography]
+- [[[eip]]] Gregor Hohpe. Enterprise Integration Patterns [online]. Retrieved: 27 Dec 2014, from: http://www.enterpriseintegrationpatterns.com/
+- [[[soa]]] Wikipedia. Service Oriented Architecture [online]. Retrieved: 27 Dec 2014, from: http://en.wikipedia.org/wiki/Service-oriented_architecture
+- [[[api]]] Eric Savitz. Welcome to the API Economy [online]. Forbes.com. Retrieved: 27 Dec 2014, from: http://www.forbes.com/sites/ciocentral/2012/08/29/welcome-to-the-api-economy/
+- [[[api2]]] Adam Duvander. The rise of the API economy and consumer-led ecosystems [online]. thenextweb.com. Retrieved: 27 Dec 2014, from: http://thenextweb.com/dd/2014/03/28/api-economy/
+- [[[iot]]] Wikipedia. Internet of Things [online]. Retrieved: 27 Dec 2014, from: http://en.wikipedia.org/wiki/Internet_of_Things
+- [[[bigdata]]] Wikipedia. Big Data [online]. Retrieved: 27 Dec 2014, from: http://en.wikipedia.org/wiki/Big_data
+- [[[fbp]]] Wikipedia. Flow Based Programming [online]. Retrieved: 28 Dec 2014, from: http://en.wikipedia.org/wiki/Flow-based_programming#Concepts
+- [[[seda]]] Matt Welsh. Harvard. SEDA: An Architecture for Highly Concurrent Server Applications [online]. Retrieved: 28 Dec 2014, from: http://www.eecs.harvard.edu/~mdw/proj/seda/
[03/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/impl/StandardUserService.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/impl/StandardUserService.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/impl/StandardUserService.java
new file mode 100644
index 0000000..424816f
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/impl/StandardUserService.java
@@ -0,0 +1,642 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.impl;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.nifi.admin.dao.DataAccessException;
+import org.apache.nifi.admin.service.AccountDisabledException;
+import org.apache.nifi.admin.service.AccountPendingException;
+import org.apache.nifi.admin.service.AdministrationException;
+import org.apache.nifi.admin.service.UserService;
+import org.apache.nifi.admin.service.action.AuthorizeDownloadAction;
+import org.apache.nifi.admin.service.action.AuthorizeUserAction;
+import org.apache.nifi.admin.service.action.DeleteUserAction;
+import org.apache.nifi.admin.service.action.DisableUserAction;
+import org.apache.nifi.admin.service.action.DisableUserGroupAction;
+import org.apache.nifi.admin.service.action.FindUserByDnAction;
+import org.apache.nifi.admin.service.action.FindUserByIdAction;
+import org.apache.nifi.admin.service.action.GetUserGroupAction;
+import org.apache.nifi.admin.service.action.GetUsersAction;
+import org.apache.nifi.admin.service.action.HasPendingUserAccounts;
+import org.apache.nifi.admin.service.action.InvalidateUserAccountAction;
+import org.apache.nifi.admin.service.action.InvalidateUserGroupAccountsAction;
+import org.apache.nifi.admin.service.action.RequestUserAccountAction;
+import org.apache.nifi.admin.service.action.SeedUserAccountsAction;
+import org.apache.nifi.admin.service.action.UpdateUserAction;
+import org.apache.nifi.admin.service.action.UpdateUserGroupAction;
+import org.apache.nifi.admin.service.action.UngroupUserAction;
+import org.apache.nifi.admin.service.action.UngroupUserGroupAction;
+import org.apache.nifi.admin.service.transaction.Transaction;
+import org.apache.nifi.admin.service.transaction.TransactionBuilder;
+import org.apache.nifi.admin.service.transaction.TransactionException;
+import org.apache.nifi.authorization.Authority;
+import org.apache.nifi.authorization.DownloadAuthorization;
+import org.apache.nifi.user.NiFiUser;
+import org.apache.nifi.user.NiFiUserGroup;
+import org.apache.nifi.util.FormatUtils;
+import org.apache.nifi.util.NiFiProperties;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ */
+public class StandardUserService implements UserService {
+
+ private static final Logger logger = LoggerFactory.getLogger(StandardUserService.class);
+
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+ private final ReentrantReadWriteLock.ReadLock readLock = lock.readLock();
+ private final ReentrantReadWriteLock.WriteLock writeLock = lock.writeLock();
+
+ private TransactionBuilder transactionBuilder;
+ private NiFiProperties properties;
+
+ /**
+ * Seed any users from the authority provider that are not already present.
+ */
+ public void seedUserAccounts() {
+ // do not seed node's user cache. when/if the node disconnects its
+ // cache will be populated lazily (as needed)
+ if (properties.isNode()) {
+ return;
+ }
+
+ Transaction transaction = null;
+ writeLock.lock();
+ try {
+ // start the transaction
+ transaction = transactionBuilder.start();
+
+ // seed the accounts
+ SeedUserAccountsAction seedUserAccounts = new SeedUserAccountsAction();
+ transaction.execute(seedUserAccounts);
+
+ // commit the transaction
+ transaction.commit();
+ } catch (AdministrationException ae) {
+ rollback(transaction);
+ throw ae;
+ } catch (Throwable t) {
+ rollback(transaction);
+ throw t;
+ } finally {
+ closeQuietly(transaction);
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public NiFiUser createPendingUserAccount(String dn, String justification) {
+ Transaction transaction = null;
+
+ writeLock.lock();
+ try {
+ // start the transaction
+ transaction = transactionBuilder.start();
+
+ // create the account request
+ RequestUserAccountAction requestUserAccount = new RequestUserAccountAction(dn, justification);
+ NiFiUser user = transaction.execute(requestUserAccount);
+
+ // commit the transaction
+ transaction.commit();
+
+ // return the nifi user
+ return user;
+ } catch (TransactionException | DataAccessException te) {
+ rollback(transaction);
+ throw new AdministrationException(te);
+ } catch (Throwable t) {
+ rollback(transaction);
+ throw t;
+ } finally {
+ closeQuietly(transaction);
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public NiFiUserGroup updateGroup(final String group, final Set<String> userIds, final Set<Authority> authorities) {
+ Transaction transaction = null;
+
+ writeLock.lock();
+ try {
+ // if user ids have been specified, invalidate the user accounts before performing
+ // the desired updates. if case of an error, this will ensure that these users are
+ // authorized the next time the access the application
+ if (userIds != null) {
+ for (final String userId : userIds) {
+ invalidateUserAccount(userId);
+ }
+ }
+
+ // start the transaction
+ transaction = transactionBuilder.start();
+
+ // set the authorities for each user in this group if specified
+ final UpdateUserGroupAction updateUserGroup = new UpdateUserGroupAction(group, userIds, authorities);
+ transaction.execute(updateUserGroup);
+
+ // get all the users that are now in this group
+ final GetUserGroupAction getUserGroup = new GetUserGroupAction(group);
+ final NiFiUserGroup userGroup = transaction.execute(getUserGroup);
+
+ // commit the transaction
+ transaction.commit();
+
+ return userGroup;
+ } catch (TransactionException | DataAccessException te) {
+ rollback(transaction);
+ throw new AdministrationException(te);
+ } catch (Throwable t) {
+ rollback(transaction);
+ throw t;
+ } finally {
+ closeQuietly(transaction);
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public void ungroupUser(String id) {
+ Transaction transaction = null;
+
+ writeLock.lock();
+ try {
+ // start the transaction
+ transaction = transactionBuilder.start();
+
+ // ungroup the specified user
+ final UngroupUserAction ungroupUser = new UngroupUserAction(id);
+ transaction.execute(ungroupUser);
+
+ // commit the transaction
+ transaction.commit();
+ } catch (TransactionException | DataAccessException te) {
+ rollback(transaction);
+ throw new AdministrationException(te);
+ } catch (Throwable t) {
+ rollback(transaction);
+ throw t;
+ } finally {
+ closeQuietly(transaction);
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public void ungroup(String group) {
+ Transaction transaction = null;
+
+ writeLock.lock();
+ try {
+ // start the transaction
+ transaction = transactionBuilder.start();
+
+ // ungroup the specified user
+ final UngroupUserGroupAction ungroupUserGroup = new UngroupUserGroupAction(group);
+ transaction.execute(ungroupUserGroup);
+
+ // commit the transaction
+ transaction.commit();
+ } catch (TransactionException | DataAccessException te) {
+ rollback(transaction);
+ throw new AdministrationException(te);
+ } catch (Throwable t) {
+ rollback(transaction);
+ throw t;
+ } finally {
+ closeQuietly(transaction);
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public NiFiUser checkAuthorization(String dn) {
+ Transaction transaction = null;
+
+ writeLock.lock();
+ try {
+ // create the connection
+ transaction = transactionBuilder.start();
+
+ // determine how long the cache is valid for
+ final int cacheSeconds;
+ try {
+ cacheSeconds = (int) FormatUtils.getTimeDuration(properties.getUserCredentialCacheDuration(), TimeUnit.SECONDS);
+ } catch (IllegalArgumentException iae) {
+ throw new AdministrationException("User credential cache duration is not configured correctly.");
+ }
+
+ // attempt to authorize the user
+ AuthorizeUserAction authorizeUser = new AuthorizeUserAction(dn, cacheSeconds);
+ NiFiUser user = transaction.execute(authorizeUser);
+
+ // commit the transaction
+ transaction.commit();
+
+ // return the nifi user
+ return user;
+ } catch (DataAccessException | TransactionException dae) {
+ rollback(transaction);
+ throw new AdministrationException(dae);
+ } catch (AccountDisabledException | AccountPendingException ade) {
+ rollback(transaction);
+ throw ade;
+ } catch (Throwable t) {
+ rollback(transaction);
+ throw t;
+ } finally {
+ closeQuietly(transaction);
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public void deleteUser(String id) {
+ Transaction transaction = null;
+
+ writeLock.lock();
+ try {
+ // create the connection
+ transaction = transactionBuilder.start();
+
+ // delete the user
+ DeleteUserAction deleteUser = new DeleteUserAction(id);
+ transaction.execute(deleteUser);
+
+ // commit the transaction
+ transaction.commit();
+ } catch (DataAccessException | TransactionException dae) {
+ rollback(transaction);
+ throw new AdministrationException(dae);
+ } catch (Throwable t) {
+ rollback(transaction);
+ throw t;
+ } finally {
+ closeQuietly(transaction);
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public NiFiUser disable(String id) {
+ Transaction transaction = null;
+
+ writeLock.lock();
+ try {
+ // create the connection
+ transaction = transactionBuilder.start();
+
+ // disable the user
+ DisableUserAction disableUser = new DisableUserAction(id);
+ NiFiUser user = transaction.execute(disableUser);
+
+ // commit the transaction
+ transaction.commit();
+
+ // return the user
+ return user;
+ } catch (DataAccessException | TransactionException dae) {
+ rollback(transaction);
+ throw new AdministrationException(dae);
+ } catch (Throwable t) {
+ rollback(transaction);
+ throw t;
+ } finally {
+ closeQuietly(transaction);
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public NiFiUserGroup disableGroup(String group) {
+ Transaction transaction = null;
+
+ writeLock.lock();
+ try {
+ // create the connection
+ transaction = transactionBuilder.start();
+
+ // disable the user
+ DisableUserGroupAction disableUser = new DisableUserGroupAction(group);
+ NiFiUserGroup userGroup = transaction.execute(disableUser);
+
+ // commit the transaction
+ transaction.commit();
+
+ // return the user
+ return userGroup;
+ } catch (DataAccessException | TransactionException dae) {
+ rollback(transaction);
+ throw new AdministrationException(dae);
+ } catch (Throwable t) {
+ rollback(transaction);
+ throw t;
+ } finally {
+ closeQuietly(transaction);
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public NiFiUser update(String id, Set<Authority> authorities) {
+ Transaction transaction = null;
+
+ // may be empty but not null
+ if (authorities == null) {
+ throw new IllegalArgumentException("The specified authorities cannot be null.");
+ }
+
+ writeLock.lock();
+ try {
+ // invalidate the user account in preparation for potential subsequent errors
+ invalidateUserAccount(id);
+
+ // at this point the current user account has been invalidated so we will
+ // attempt to update the account. if any part fails we are assured the
+ // user will be need to be given approval before they access the system at
+ // a later time
+ // start the transaction
+ transaction = transactionBuilder.start();
+
+ // update the user authorities
+ UpdateUserAction setUserAuthorities = new UpdateUserAction(id, authorities);
+ NiFiUser user = transaction.execute(setUserAuthorities);
+
+ // commit the transaction
+ transaction.commit();
+
+ // return the user
+ return user;
+ } catch (TransactionException | DataAccessException e) {
+ rollback(transaction);
+ throw new AdministrationException(e);
+ } catch (Throwable t) {
+ rollback(transaction);
+ throw t;
+ } finally {
+ closeQuietly(transaction);
+ writeLock.unlock();
+ }
+ }
+
+ /**
+ * Invalidates the user with the specified id. This is done to ensure a user
+ * account will need to be re-validated in case an error occurs while
+ * modifying a user account. This method should only be invoked from within
+ * a write lock.
+ *
+ * @param id user account identifier
+ */
+ @Override
+ public void invalidateUserAccount(String id) {
+ Transaction transaction = null;
+
+ writeLock.lock();
+ try {
+ // start the transaction
+ transaction = transactionBuilder.start();
+
+ // invalidate the user account
+ InvalidateUserAccountAction invalidateUserAccount = new InvalidateUserAccountAction(id);
+ transaction.execute(invalidateUserAccount);
+
+ // commit the transaction
+ transaction.commit();
+ } catch (TransactionException | DataAccessException te) {
+ rollback(transaction);
+ throw new AdministrationException(te);
+ } catch (Throwable t) {
+ rollback(transaction);
+ throw t;
+ } finally {
+ closeQuietly(transaction);
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public void invalidateUserGroupAccount(String group) {
+ Transaction transaction = null;
+
+ writeLock.lock();
+ try {
+ // start the transaction
+ transaction = transactionBuilder.start();
+
+ // invalidate the user account
+ InvalidateUserGroupAccountsAction invalidateUserGroupAccounts = new InvalidateUserGroupAccountsAction(group);
+ transaction.execute(invalidateUserGroupAccounts);
+
+ // commit the transaction
+ transaction.commit();
+ } catch (TransactionException | DataAccessException te) {
+ rollback(transaction);
+ throw new AdministrationException(te);
+ } catch (Throwable t) {
+ rollback(transaction);
+ throw t;
+ } finally {
+ closeQuietly(transaction);
+ writeLock.unlock();
+ }
+ }
+
+ // -----------------
+ // read only methods
+ // -----------------
+ @Override
+ public Boolean hasPendingUserAccount() {
+ Transaction transaction = null;
+
+ readLock.lock();
+ try {
+ // start the transaction
+ transaction = transactionBuilder.start();
+
+ final HasPendingUserAccounts hasPendingAccounts = new HasPendingUserAccounts();
+ final Boolean hasPendingUserAccounts = transaction.execute(hasPendingAccounts);
+
+ // commit the transaction
+ transaction.commit();
+
+ return hasPendingUserAccounts;
+ } catch (TransactionException | DataAccessException te) {
+ rollback(transaction);
+ throw new AdministrationException(te);
+ } catch (Throwable t) {
+ rollback(transaction);
+ throw t;
+ } finally {
+ closeQuietly(transaction);
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public DownloadAuthorization authorizeDownload(final List<String> dnChain, final Map<String, String> attributes) {
+ Transaction transaction = null;
+
+ readLock.lock();
+ try {
+ // start the transaction
+ transaction = transactionBuilder.start();
+
+ // authorize the download
+ AuthorizeDownloadAction authorizeDownload = new AuthorizeDownloadAction(dnChain, attributes);
+ DownloadAuthorization downloadAuthorization = transaction.execute(authorizeDownload);
+
+ // commit the transaction
+ transaction.commit();
+
+ // return the authorization
+ return downloadAuthorization;
+ } catch (TransactionException | DataAccessException te) {
+ rollback(transaction);
+ throw new AdministrationException(te);
+ } catch (Throwable t) {
+ rollback(transaction);
+ throw t;
+ } finally {
+ closeQuietly(transaction);
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public Collection<NiFiUser> getUsers() {
+ Transaction transaction = null;
+
+ readLock.lock();
+ try {
+ // start the transaction
+ transaction = transactionBuilder.start();
+
+ // get all users
+ GetUsersAction getUsers = new GetUsersAction();
+ Collection<NiFiUser> users = transaction.execute(getUsers);
+
+ // commit the transaction
+ transaction.commit();
+
+ // return the users
+ return users;
+ } catch (TransactionException | DataAccessException te) {
+ rollback(transaction);
+ throw new AdministrationException(te);
+ } catch (Throwable t) {
+ rollback(transaction);
+ throw t;
+ } finally {
+ closeQuietly(transaction);
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public NiFiUser getUserById(String id) {
+ Transaction transaction = null;
+
+ readLock.lock();
+ try {
+ // start the transaction
+ transaction = transactionBuilder.start();
+
+ // return the desired user
+ FindUserByIdAction findUserById = new FindUserByIdAction(id);
+ NiFiUser user = transaction.execute(findUserById);
+
+ // commit the transaction
+ transaction.commit();
+
+ // return the user
+ return user;
+ } catch (TransactionException | DataAccessException te) {
+ rollback(transaction);
+ throw new AdministrationException(te);
+ } catch (Throwable t) {
+ rollback(transaction);
+ throw t;
+ } finally {
+ closeQuietly(transaction);
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public NiFiUser getUserByDn(String dn) {
+ Transaction transaction = null;
+
+ readLock.lock();
+ try {
+ // start the transaction
+ transaction = transactionBuilder.start();
+
+ // return the desired user
+ FindUserByDnAction findUserByDn = new FindUserByDnAction(dn);
+ NiFiUser user = transaction.execute(findUserByDn);
+
+ // commit the transaction
+ transaction.commit();
+
+ // return the user
+ return user;
+ } catch (TransactionException | DataAccessException te) {
+ rollback(transaction);
+ throw new AdministrationException(te);
+ } catch (Throwable t) {
+ rollback(transaction);
+ throw t;
+ } finally {
+ closeQuietly(transaction);
+ readLock.unlock();
+ }
+ }
+
+ private void rollback(final Transaction transaction) {
+ if (transaction != null) {
+ transaction.rollback();
+ }
+ }
+
+ private void closeQuietly(final Transaction transaction) {
+ if (transaction != null) {
+ try {
+ transaction.close();
+ } catch (final IOException ioe) {
+ }
+ }
+ }
+
+ public void setTransactionBuilder(TransactionBuilder transactionBuilder) {
+ this.transactionBuilder = transactionBuilder;
+ }
+
+ public void setProperties(NiFiProperties properties) {
+ this.properties = properties;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/transaction/Transaction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/transaction/Transaction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/transaction/Transaction.java
new file mode 100644
index 0000000..4792581
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/transaction/Transaction.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.transaction;
+
+import java.io.Closeable;
+import org.apache.nifi.admin.service.action.AdministrationAction;
+
+/**
+ * Defines a transaction.
+ */
+public interface Transaction extends Closeable {
+
+ /**
+ * Executes the specified action within the current transaction.
+ *
+ * @param <T> type of action to execute
+ * @param action action to execute
+ * @return executed action
+ * @throws IllegalStateException - if there is no current transaction
+ */
+ <T> T execute(AdministrationAction<T> action);
+
+ /**
+ * Commits the current transaction.
+ *
+ * @throws TransactionException - if the transaction is unable to be
+ * committed
+ */
+ void commit() throws TransactionException;
+
+ /**
+ * Rolls back the current transaction.
+ */
+ void rollback();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/transaction/TransactionBuilder.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/transaction/TransactionBuilder.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/transaction/TransactionBuilder.java
new file mode 100644
index 0000000..2d2ef82
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/transaction/TransactionBuilder.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.transaction;
+
+/**
+ *
+ */
+public interface TransactionBuilder {
+
+ Transaction start() throws TransactionException;
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/transaction/TransactionException.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/transaction/TransactionException.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/transaction/TransactionException.java
new file mode 100644
index 0000000..924e01f
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/transaction/TransactionException.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.transaction;
+
+/**
+ * Exception to indicate that the user account is disabled.
+ */
+public class TransactionException extends RuntimeException {
+
+ public TransactionException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ }
+
+ public TransactionException(Throwable cause) {
+ super(cause);
+ }
+
+ public TransactionException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public TransactionException(String message) {
+ super(message);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/transaction/impl/StandardTransaction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/transaction/impl/StandardTransaction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/transaction/impl/StandardTransaction.java
new file mode 100644
index 0000000..a3cfb5e
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/transaction/impl/StandardTransaction.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.transaction.impl;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.SQLException;
+import org.apache.nifi.admin.RepositoryUtils;
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.admin.dao.impl.DAOFactoryImpl;
+import org.apache.nifi.admin.service.action.AdministrationAction;
+import org.apache.nifi.admin.service.transaction.TransactionException;
+import org.apache.nifi.admin.service.transaction.Transaction;
+import org.apache.nifi.authorization.AuthorityProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Transaction implementation that uses the specified SQL Connection and
+ * AuthorityProvider.
+ */
+public class StandardTransaction implements Transaction {
+
+ private static final Logger logger = LoggerFactory.getLogger(StandardTransaction.class);
+
+ private final AuthorityProvider authorityProvider;
+ private Connection connection;
+
+ public StandardTransaction(AuthorityProvider authorityProvider, Connection connection) {
+ this.authorityProvider = authorityProvider;
+ this.connection = connection;
+ }
+
+ @Override
+ public <T> T execute(AdministrationAction<T> action) {
+ // ensure the transaction has been started
+ if (connection == null) {
+ throw new IllegalStateException("This transaction is not active.");
+ }
+
+ // create a dao factory
+ DAOFactory daoFactory = new DAOFactoryImpl(connection);
+
+ // execute the specified action
+ return action.execute(daoFactory, authorityProvider);
+ }
+
+ @Override
+ public void commit() throws TransactionException {
+ // ensure there is an active transaction
+ if (connection == null) {
+ throw new IllegalStateException("No active transaction.");
+ }
+
+ try {
+ // commit the transaction
+ connection.commit();
+ } catch (SQLException sqle) {
+ throw new TransactionException(sqle.getMessage());
+ }
+ }
+
+ @Override
+ public void rollback() {
+ // ensure there is an active transaction
+ if (connection != null) {
+ // rollback the transaction
+ RepositoryUtils.rollback(connection, logger);
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (connection != null) {
+ RepositoryUtils.closeQuietly(connection);
+ connection = null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/transaction/impl/StandardTransactionBuilder.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/transaction/impl/StandardTransactionBuilder.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/transaction/impl/StandardTransactionBuilder.java
new file mode 100644
index 0000000..b6e5a30
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/transaction/impl/StandardTransactionBuilder.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.transaction.impl;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import javax.sql.DataSource;
+import org.apache.nifi.admin.service.transaction.Transaction;
+import org.apache.nifi.admin.service.transaction.TransactionBuilder;
+import org.apache.nifi.admin.service.transaction.TransactionException;
+import org.apache.nifi.authorization.AuthorityProvider;
+
+/**
+ *
+ */
+public class StandardTransactionBuilder implements TransactionBuilder {
+
+ private DataSource dataSource;
+ private AuthorityProvider authorityProvider;
+
+ @Override
+ public Transaction start() throws TransactionException {
+ try {
+ // get a new connection
+ Connection connection = dataSource.getConnection();
+ connection.setAutoCommit(false);
+
+ // create a new transaction
+ return new StandardTransaction(authorityProvider, connection);
+ } catch (SQLException sqle) {
+ throw new TransactionException(sqle.getMessage());
+ }
+ }
+
+ /* setters */
+ public void setDataSource(DataSource dataSource) {
+ this.dataSource = dataSource;
+ }
+
+ public void setAuthorityProvider(AuthorityProvider authorityProvider) {
+ this.authorityProvider = authorityProvider;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/authorization/AuthorityProviderFactoryBean.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/authorization/AuthorityProviderFactoryBean.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/authorization/AuthorityProviderFactoryBean.java
new file mode 100644
index 0000000..b3e9547
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/authorization/AuthorityProviderFactoryBean.java
@@ -0,0 +1,490 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.authorization;
+
+import java.io.File;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import javax.xml.XMLConstants;
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBElement;
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.Unmarshaller;
+import javax.xml.transform.stream.StreamSource;
+import javax.xml.validation.Schema;
+import javax.xml.validation.SchemaFactory;
+import org.apache.nifi.authorization.annotation.AuthorityProviderContext;
+import org.apache.nifi.authorization.exception.AuthorityAccessException;
+import org.apache.nifi.authorization.exception.IdentityAlreadyExistsException;
+import org.apache.nifi.authorization.exception.ProviderCreationException;
+import org.apache.nifi.authorization.exception.ProviderDestructionException;
+import org.apache.nifi.authorization.exception.UnknownIdentityException;
+import org.apache.nifi.authorization.generated.AuthorityProviders;
+import org.apache.nifi.authorization.generated.Property;
+import org.apache.nifi.authorization.generated.Provider;
+import org.apache.nifi.nar.ExtensionManager;
+import org.apache.nifi.nar.NarCloseable;
+import org.apache.nifi.util.NiFiProperties;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.BeansException;
+import org.springframework.beans.factory.DisposableBean;
+import org.springframework.beans.factory.FactoryBean;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.ApplicationContextAware;
+import org.xml.sax.SAXException;
+
+/**
+ * Factory bean for loading the configured authority provider.
+ */
+public class AuthorityProviderFactoryBean implements FactoryBean, ApplicationContextAware, DisposableBean, AuthorityProviderLookup {
+
+ private static final Logger logger = LoggerFactory.getLogger(AuthorityProviderFactoryBean.class);
+ private static final String AUTHORITY_PROVIDERS_XSD = "/authority-providers.xsd";
+ private static final String JAXB_GENERATED_PATH = "org.apache.nifi.authorization.generated";
+ private static final JAXBContext JAXB_CONTEXT = initializeJaxbContext();
+
+ /**
+ * Load the JAXBContext.
+ */
+ private static JAXBContext initializeJaxbContext() {
+ try {
+ return JAXBContext.newInstance(JAXB_GENERATED_PATH, AuthorityProviderFactoryBean.class.getClassLoader());
+ } catch (JAXBException e) {
+ throw new RuntimeException("Unable to create JAXBContext.");
+ }
+ }
+
+ private ApplicationContext applicationContext;
+ private AuthorityProvider authorityProvider;
+ private NiFiProperties properties;
+ private final Map<String, AuthorityProvider> authorityProviders = new HashMap<>();
+
+ @Override
+ public AuthorityProvider getAuthorityProvider(String identifier) {
+ return authorityProviders.get(identifier);
+ }
+
+ @Override
+ public Object getObject() throws Exception {
+ if (authorityProvider == null) {
+ // look up the authority provider to use
+ final String authorityProviderIdentifier = properties.getProperty(NiFiProperties.SECURITY_USER_AUTHORITY_PROVIDER);
+
+ // ensure the authority provider class name was specified
+ if (StringUtils.isBlank(authorityProviderIdentifier)) {
+ // if configured for ssl, the authority provider must be specified
+ if (properties.getSslPort() != null) {
+ throw new Exception("When running securely, the authority provider identifier must be specified in the nifi properties file.");
+ }
+
+ // use a default provider... only allowable when running not securely
+ authorityProvider = createDefaultProvider();
+ } else {
+ final AuthorityProviders authorityProviderConfiguration = loadAuthorityProvidersConfiguration();
+
+ // create each authority provider
+ for (final Provider provider : authorityProviderConfiguration.getProvider()) {
+ authorityProviders.put(provider.getIdentifier(), createAuthorityProvider(provider.getIdentifier(), provider.getClazz()));
+ }
+
+ // configure each authority provider
+ for (final Provider provider : authorityProviderConfiguration.getProvider()) {
+ final AuthorityProvider instance = authorityProviders.get(provider.getIdentifier());
+ instance.onConfigured(loadAuthorityProviderConfiguration(provider));
+ }
+
+ // get the authority provider instance
+ authorityProvider = getAuthorityProvider(authorityProviderIdentifier);
+
+ // ensure it was found
+ if (authorityProvider == null) {
+ throw new Exception(String.format("The specified authority provider '%s' could not be found.", authorityProviderIdentifier));
+ }
+ }
+ }
+
+ return authorityProvider;
+ }
+
+ private AuthorityProviders loadAuthorityProvidersConfiguration() throws Exception {
+ final File authorityProvidersConfigurationFile = properties.getAuthorityProviderConfiguraitonFile();
+
+ // load the users from the specified file
+ if (authorityProvidersConfigurationFile.exists()) {
+ try {
+ // find the schema
+ final SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
+ final Schema schema = schemaFactory.newSchema(AuthorityProviders.class.getResource(AUTHORITY_PROVIDERS_XSD));
+
+ // attempt to unmarshal
+ final Unmarshaller unmarshaller = JAXB_CONTEXT.createUnmarshaller();
+ unmarshaller.setSchema(schema);
+ final JAXBElement<AuthorityProviders> element = unmarshaller.unmarshal(new StreamSource(authorityProvidersConfigurationFile), AuthorityProviders.class);
+ return element.getValue();
+ } catch (SAXException | JAXBException e) {
+ throw new Exception("Unable to load the authority provider configuration file at: " + authorityProvidersConfigurationFile.getAbsolutePath());
+ }
+ } else {
+ throw new Exception("Unable to find the authority provider configuration file at " + authorityProvidersConfigurationFile.getAbsolutePath());
+ }
+ }
+
+ private AuthorityProvider createAuthorityProvider(final String identifier, final String authorityProviderClassName) throws Exception {
+ // get the classloader for the specified authority provider
+ final ClassLoader authorityProviderClassLoader = ExtensionManager.getClassLoader(authorityProviderClassName);
+ if (authorityProviderClassLoader == null) {
+ throw new Exception(String.format("The specified authority provider class '%s' is not known to this nifi.", authorityProviderClassName));
+ }
+
+ // get the current context classloader
+ final ClassLoader currentClassLoader = Thread.currentThread().getContextClassLoader();
+
+ final AuthorityProvider instance;
+ try {
+ // set the appropriate class loader
+ Thread.currentThread().setContextClassLoader(authorityProviderClassLoader);
+
+ // attempt to load the class
+ Class<?> rawAuthorityProviderClass = Class.forName(authorityProviderClassName, true, authorityProviderClassLoader);
+ Class<? extends AuthorityProvider> authorityProviderClass = rawAuthorityProviderClass.asSubclass(AuthorityProvider.class);
+
+ // otherwise create a new instance
+ Constructor constructor = authorityProviderClass.getConstructor();
+ instance = (AuthorityProvider) constructor.newInstance();
+
+ // method injection
+ performMethodInjection(instance, authorityProviderClass);
+
+ // field injection
+ performFieldInjection(instance, authorityProviderClass);
+
+ // call post construction lifecycle event
+ instance.initialize(new StandardAuthorityProviderInitializationContext(identifier, this));
+ } finally {
+ if (currentClassLoader != null) {
+ Thread.currentThread().setContextClassLoader(currentClassLoader);
+ }
+ }
+
+ return withNarLoader(instance);
+ }
+
+ private AuthorityProviderConfigurationContext loadAuthorityProviderConfiguration(final Provider provider) {
+ final Map<String, String> providerProperties = new HashMap<>();
+
+ for (final Property property : provider.getProperty()) {
+ providerProperties.put(property.getName(), property.getValue());
+ }
+
+ return new StandardAuthorityProviderConfigurationContext(provider.getIdentifier(), providerProperties);
+ }
+
+ private void performMethodInjection(final AuthorityProvider instance, final Class authorityProviderClass) throws IllegalAccessException, IllegalArgumentException, InvocationTargetException {
+ for (final Method method : authorityProviderClass.getMethods()) {
+ if (method.isAnnotationPresent(AuthorityProviderContext.class)) {
+ // make the method accessible
+ final boolean isAccessible = method.isAccessible();
+ method.setAccessible(true);
+
+ try {
+ final Class<?>[] argumentTypes = method.getParameterTypes();
+
+ // look for setters (single argument)
+ if (argumentTypes.length == 1) {
+ final Class<?> argumentType = argumentTypes[0];
+
+ // look for well known types
+ if (NiFiProperties.class.isAssignableFrom(argumentType)) {
+ // nifi properties injection
+ method.invoke(instance, properties);
+ } else if (ApplicationContext.class.isAssignableFrom(argumentType)) {
+ // spring application context injection
+ method.invoke(instance, applicationContext);
+ }
+ }
+ } finally {
+ method.setAccessible(isAccessible);
+ }
+ }
+ }
+
+ final Class parentClass = authorityProviderClass.getSuperclass();
+ if (parentClass != null && AuthorityProvider.class.isAssignableFrom(parentClass)) {
+ performMethodInjection(instance, parentClass);
+ }
+ }
+
+ private void performFieldInjection(final AuthorityProvider instance, final Class authorityProviderClass) throws IllegalArgumentException, IllegalAccessException {
+ for (final Field field : authorityProviderClass.getDeclaredFields()) {
+ if (field.isAnnotationPresent(AuthorityProviderContext.class)) {
+ // make the method accessible
+ final boolean isAccessible = field.isAccessible();
+ field.setAccessible(true);
+
+ try {
+ // get the type
+ final Class<?> fieldType = field.getType();
+
+ // only consider this field if it isn't set yet
+ if (field.get(instance) == null) {
+ // look for well known types
+ if (NiFiProperties.class.isAssignableFrom(fieldType)) {
+ // nifi properties injection
+ field.set(instance, properties);
+ } else if (ApplicationContext.class.isAssignableFrom(fieldType)) {
+ // spring application context injection
+ field.set(instance, applicationContext);
+ }
+ }
+
+ } finally {
+ field.setAccessible(isAccessible);
+ }
+ }
+ }
+
+ final Class parentClass = authorityProviderClass.getSuperclass();
+ if (parentClass != null && AuthorityProvider.class.isAssignableFrom(parentClass)) {
+ performFieldInjection(instance, parentClass);
+ }
+ }
+
+ /**
+ * @return a default provider to use when running unsecurely with no
+ * provider configured
+ */
+ private AuthorityProvider createDefaultProvider() {
+ return new AuthorityProvider() {
+ @Override
+ public boolean doesDnExist(String dn) throws AuthorityAccessException {
+ return false;
+ }
+
+ @Override
+ public Set<Authority> getAuthorities(String dn) throws UnknownIdentityException, AuthorityAccessException {
+ return EnumSet.noneOf(Authority.class);
+ }
+
+ @Override
+ public void setAuthorities(String dn, Set<Authority> authorities) throws UnknownIdentityException, AuthorityAccessException {
+ }
+
+ @Override
+ public Set<String> getUsers(Authority authority) throws AuthorityAccessException {
+ return new HashSet<>();
+ }
+
+ @Override
+ public void revokeUser(String dn) throws UnknownIdentityException, AuthorityAccessException {
+ }
+
+ @Override
+ public void addUser(String dn, String group) throws IdentityAlreadyExistsException, AuthorityAccessException {
+ }
+
+ @Override
+ public String getGroupForUser(String dn) throws UnknownIdentityException, AuthorityAccessException {
+ return null;
+ }
+
+ @Override
+ public void revokeGroup(String group) throws UnknownIdentityException, AuthorityAccessException {
+ }
+
+ @Override
+ public void setUsersGroup(Set<String> dn, String group) throws UnknownIdentityException, AuthorityAccessException {
+ }
+
+ @Override
+ public void ungroupUser(String dn) throws UnknownIdentityException, AuthorityAccessException {
+ }
+
+ @Override
+ public void ungroup(String group) throws AuthorityAccessException {
+ }
+
+ @Override
+ public DownloadAuthorization authorizeDownload(List<String> dnChain, Map<String, String> attributes) throws UnknownIdentityException, AuthorityAccessException {
+ return DownloadAuthorization.approved();
+ }
+
+ @Override
+ public void initialize(AuthorityProviderInitializationContext initializationContext) throws ProviderCreationException {
+ }
+
+ @Override
+ public void onConfigured(AuthorityProviderConfigurationContext configurationContext) throws ProviderCreationException {
+ }
+
+ @Override
+ public void preDestruction() throws ProviderDestructionException {
+ }
+ };
+ }
+
+ /**
+ * Decorates the base provider to ensure the nar context classloader is used
+ * when invoking the underlying methods.
+ *
+ * @param baseProvider base provider
+ * @return provider
+ */
+ public AuthorityProvider withNarLoader(final AuthorityProvider baseProvider) {
+ return new AuthorityProvider() {
+ @Override
+ public boolean doesDnExist(String dn) throws AuthorityAccessException {
+ try (final NarCloseable narCloseable = NarCloseable.withNarLoader()) {
+ return baseProvider.doesDnExist(dn);
+ }
+ }
+
+ @Override
+ public Set<Authority> getAuthorities(String dn) throws UnknownIdentityException, AuthorityAccessException {
+ try (final NarCloseable narCloseable = NarCloseable.withNarLoader()) {
+ return baseProvider.getAuthorities(dn);
+ }
+ }
+
+ @Override
+ public void setAuthorities(String dn, Set<Authority> authorities) throws UnknownIdentityException, AuthorityAccessException {
+ try (final NarCloseable narCloseable = NarCloseable.withNarLoader()) {
+ baseProvider.setAuthorities(dn, authorities);
+ }
+ }
+
+ @Override
+ public Set<String> getUsers(Authority authority) throws AuthorityAccessException {
+ try (final NarCloseable narCloseable = NarCloseable.withNarLoader()) {
+ return baseProvider.getUsers(authority);
+ }
+ }
+
+ @Override
+ public void revokeUser(String dn) throws UnknownIdentityException, AuthorityAccessException {
+ try (final NarCloseable narCloseable = NarCloseable.withNarLoader()) {
+ baseProvider.revokeUser(dn);
+ }
+ }
+
+ @Override
+ public void addUser(String dn, String group) throws IdentityAlreadyExistsException, AuthorityAccessException {
+ try (final NarCloseable narCloseable = NarCloseable.withNarLoader()) {
+ baseProvider.addUser(dn, group);
+ }
+ }
+
+ @Override
+ public String getGroupForUser(String dn) throws UnknownIdentityException, AuthorityAccessException {
+ try (final NarCloseable narCloseable = NarCloseable.withNarLoader()) {
+ return baseProvider.getGroupForUser(dn);
+ }
+ }
+
+ @Override
+ public void revokeGroup(String group) throws UnknownIdentityException, AuthorityAccessException {
+ try (final NarCloseable narCloseable = NarCloseable.withNarLoader()) {
+ baseProvider.revokeGroup(group);
+ }
+ }
+
+ @Override
+ public void setUsersGroup(Set<String> dns, String group) throws UnknownIdentityException, AuthorityAccessException {
+ try (final NarCloseable narCloseable = NarCloseable.withNarLoader()) {
+ baseProvider.setUsersGroup(dns, group);
+ }
+ }
+
+ @Override
+ public void ungroupUser(String dn) throws UnknownIdentityException, AuthorityAccessException {
+ try (final NarCloseable narCloseable = NarCloseable.withNarLoader()) {
+ baseProvider.ungroupUser(dn);
+ }
+ }
+
+ @Override
+ public void ungroup(String group) throws AuthorityAccessException {
+ try (final NarCloseable narCloseable = NarCloseable.withNarLoader()) {
+ baseProvider.ungroup(group);
+ }
+ }
+
+ @Override
+ public DownloadAuthorization authorizeDownload(List<String> dnChain, Map<String, String> attributes) throws UnknownIdentityException, AuthorityAccessException {
+ try (final NarCloseable narCloseable = NarCloseable.withNarLoader()) {
+ return baseProvider.authorizeDownload(dnChain, attributes);
+ }
+ }
+
+ @Override
+ public void initialize(AuthorityProviderInitializationContext initializationContext) throws ProviderCreationException {
+ try (final NarCloseable narCloseable = NarCloseable.withNarLoader()) {
+ baseProvider.initialize(initializationContext);
+ }
+ }
+
+ @Override
+ public void onConfigured(AuthorityProviderConfigurationContext configurationContext) throws ProviderCreationException {
+ try (final NarCloseable narCloseable = NarCloseable.withNarLoader()) {
+ baseProvider.onConfigured(configurationContext);
+ }
+ }
+
+ @Override
+ public void preDestruction() throws ProviderDestructionException {
+ try (final NarCloseable narCloseable = NarCloseable.withNarLoader()) {
+ baseProvider.preDestruction();
+ }
+ }
+ };
+ }
+
+ @Override
+ public Class getObjectType() {
+ return AuthorityProvider.class;
+ }
+
+ @Override
+ public boolean isSingleton() {
+ return true;
+ }
+
+ @Override
+ public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
+ this.applicationContext = applicationContext;
+ }
+
+ @Override
+ public void destroy() throws Exception {
+ if (authorityProvider != null) {
+ authorityProvider.preDestruction();
+ }
+ }
+
+ public void setProperties(NiFiProperties properties) {
+ this.properties = properties;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/authorization/StandardAuthorityProviderConfigurationContext.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/authorization/StandardAuthorityProviderConfigurationContext.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/authorization/StandardAuthorityProviderConfigurationContext.java
new file mode 100644
index 0000000..0535e27
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/authorization/StandardAuthorityProviderConfigurationContext.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.authorization;
+
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ *
+ */
+public class StandardAuthorityProviderConfigurationContext implements AuthorityProviderConfigurationContext {
+
+ private final String identifier;
+ private final Map<String, String> properties;
+
+ public StandardAuthorityProviderConfigurationContext(String identifier, Map<String, String> properties) {
+ this.identifier = identifier;
+ this.properties = properties;
+ }
+
+ @Override
+ public String getIdentifier() {
+ return identifier;
+ }
+
+ @Override
+ public Map<String, String> getProperties() {
+ return Collections.unmodifiableMap(properties);
+ }
+
+ @Override
+ public String getProperty(String property) {
+ return properties.get(property);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/authorization/StandardAuthorityProviderInitializationContext.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/authorization/StandardAuthorityProviderInitializationContext.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/authorization/StandardAuthorityProviderInitializationContext.java
new file mode 100644
index 0000000..e4b16c4
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/authorization/StandardAuthorityProviderInitializationContext.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.authorization;
+
+/**
+ *
+ */
+public class StandardAuthorityProviderInitializationContext implements AuthorityProviderInitializationContext {
+
+ private final String identifier;
+ private final AuthorityProviderLookup authorityProviderLookup;
+
+ public StandardAuthorityProviderInitializationContext(String identifier, AuthorityProviderLookup authorityProviderLookup) {
+ this.identifier = identifier;
+ this.authorityProviderLookup = authorityProviderLookup;
+ }
+
+ @Override
+ public String getIdentifier() {
+ return identifier;
+ }
+
+ @Override
+ public AuthorityProviderLookup getAuthorityProviderLookup() {
+ return authorityProviderLookup;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/history/History.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/history/History.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/history/History.java
new file mode 100644
index 0000000..8536871
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/history/History.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.history;
+
+import java.util.Collection;
+import java.util.Date;
+import org.apache.nifi.action.Action;
+
+/**
+ * The result of running an action query.
+ */
+public class History {
+
+ private Integer total;
+ private Date lastRefreshed;
+ private Collection<Action> actions;
+
+ public Collection<Action> getActions() {
+ return actions;
+ }
+
+ public void setActions(Collection<Action> actions) {
+ this.actions = actions;
+ }
+
+ public Integer getTotal() {
+ return total;
+ }
+
+ public void setTotal(Integer totalRecordCount) {
+ this.total = totalRecordCount;
+ }
+
+ public Date getLastRefreshed() {
+ return lastRefreshed;
+ }
+
+ public void setLastRefreshed(Date lastRefreshed) {
+ this.lastRefreshed = lastRefreshed;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/history/HistoryQuery.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/history/HistoryQuery.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/history/HistoryQuery.java
new file mode 100644
index 0000000..53cc13c
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/history/HistoryQuery.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.history;
+
+import java.util.Date;
+
+/**
+ *
+ */
+public class HistoryQuery {
+
+ private String userName;
+ private String sourceId;
+ private Date startDate;
+ private Date endDate;
+ private Integer offset;
+ private Integer count;
+ private String sortColumn;
+ private String sortOrder;
+
+ public Date getEndDate() {
+ return endDate;
+ }
+
+ public void setEndDate(Date endDate) {
+ this.endDate = endDate;
+ }
+
+ public String getSourceId() {
+ return sourceId;
+ }
+
+ public void setSourceId(String sourceId) {
+ this.sourceId = sourceId;
+ }
+
+ public Date getStartDate() {
+ return startDate;
+ }
+
+ public void setStartDate(Date startDate) {
+ this.startDate = startDate;
+ }
+
+ public String getUserName() {
+ return userName;
+ }
+
+ public void setUserName(String userName) {
+ this.userName = userName;
+ }
+
+ public Integer getOffset() {
+ return offset;
+ }
+
+ public void setOffset(Integer offset) {
+ this.offset = offset;
+ }
+
+ public Integer getCount() {
+ return count;
+ }
+
+ public void setCount(Integer count) {
+ this.count = count;
+ }
+
+ public String getSortColumn() {
+ return sortColumn;
+ }
+
+ public void setSortColumn(String sortColumn) {
+ this.sortColumn = sortColumn;
+ }
+
+ public String getSortOrder() {
+ return sortOrder;
+ }
+
+ public void setSortOrder(String sortOrder) {
+ this.sortOrder = sortOrder;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/history/PreviousValue.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/history/PreviousValue.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/history/PreviousValue.java
new file mode 100644
index 0000000..6ece5cf
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/history/PreviousValue.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.history;
+
+import java.util.Date;
+
+/**
+ *
+ */
+public class PreviousValue {
+
+ private String previousValue;
+ private Date timestamp;
+ private String userName;
+
+ public String getPreviousValue() {
+ return previousValue;
+ }
+
+ public void setPreviousValue(String previousValue) {
+ this.previousValue = previousValue;
+ }
+
+ public Date getTimestamp() {
+ return timestamp;
+ }
+
+ public void setTimestamp(Date timestamp) {
+ this.timestamp = timestamp;
+ }
+
+ public String getUserName() {
+ return userName;
+ }
+
+ public void setUserName(String userName) {
+ this.userName = userName;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/user/AccountStatus.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/user/AccountStatus.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/user/AccountStatus.java
new file mode 100644
index 0000000..d7becf1
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/user/AccountStatus.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.user;
+
+/**
+ * Represents the status of a user's account.
+ */
+public enum AccountStatus {
+
+ ACTIVE,
+ PENDING,
+ DISABLED;
+
+ /**
+ * Returns the matching status or null if the specified status does not
+ * match any statuses.
+ *
+ * @param rawStatus string form of status
+ * @return account status object
+ */
+ public static AccountStatus valueOfStatus(String rawStatus) {
+ AccountStatus desiredStatus = null;
+
+ for (AccountStatus status : values()) {
+ if (status.toString().equals(rawStatus)) {
+ desiredStatus = status;
+ break;
+ }
+ }
+
+ return desiredStatus;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/user/NiFiUser.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/user/NiFiUser.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/user/NiFiUser.java
new file mode 100644
index 0000000..a47bde9
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/user/NiFiUser.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.user;
+
+import java.io.Serializable;
+import java.util.Date;
+import java.util.EnumSet;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.nifi.authorization.Authority;
+import org.apache.commons.lang3.StringUtils;
+
+/**
+ * An NiFiUser.
+ */
+public class NiFiUser implements Serializable {
+
+ public static final String ANONYMOUS_USER_DN = "anonymous";
+
+ private String id;
+ private String dn;
+ private String userName;
+ private String userGroup;
+ private String justification;
+
+ private Date creation;
+ private Date lastVerified;
+ private Date lastAccessed;
+
+ private AccountStatus status;
+ private EnumSet<Authority> authorities;
+
+ private NiFiUser chain;
+
+ /* getters / setters */
+ public Date getCreation() {
+ return creation;
+ }
+
+ public void setCreation(Date creation) {
+ this.creation = creation;
+ }
+
+ public String getDn() {
+ return dn;
+ }
+
+ public void setDn(String dn) {
+ this.dn = dn;
+ }
+
+ public String getUserName() {
+ return userName;
+ }
+
+ public void setUserName(String userName) {
+ this.userName = userName;
+ }
+
+ public String getUserGroup() {
+ return userGroup;
+ }
+
+ public void setUserGroup(String userGroup) {
+ this.userGroup = userGroup;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ public String getJustification() {
+ return justification;
+ }
+
+ public void setJustification(String justification) {
+ this.justification = justification;
+ }
+
+ public AccountStatus getStatus() {
+ return status;
+ }
+
+ public void setStatus(AccountStatus status) {
+ this.status = status;
+ }
+
+ public Date getLastVerified() {
+ return lastVerified;
+ }
+
+ public void setLastVerified(Date lastVerified) {
+ this.lastVerified = lastVerified;
+ }
+
+ public Date getLastAccessed() {
+ return lastAccessed;
+ }
+
+ public void setLastAccessed(Date lastAccessed) {
+ this.lastAccessed = lastAccessed;
+ }
+
+ public NiFiUser getChain() {
+ return chain;
+ }
+
+ public void setChain(NiFiUser chain) {
+ this.chain = chain;
+ }
+
+ public Set<Authority> getAuthorities() {
+ if (authorities == null) {
+ authorities = EnumSet.noneOf(Authority.class);
+ }
+ return authorities;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ final NiFiUser other = (NiFiUser) obj;
+ if (!Objects.equals(this.dn, other.dn)) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int hash = 7;
+ hash = 53 * hash + Objects.hashCode(this.dn);
+ return hash;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("dn[%s], userName[%s], justification[%s], authorities[%s]", getDn(), getUserName(), getJustification(), StringUtils.join(getAuthorities(), ", "));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/user/NiFiUserGroup.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/user/NiFiUserGroup.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/user/NiFiUserGroup.java
new file mode 100644
index 0000000..7586fd1
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/user/NiFiUserGroup.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.user;
+
+import java.util.Set;
+
+/**
+ *
+ */
+public class NiFiUserGroup {
+
+ private String group;
+ private Set<NiFiUser> users;
+
+ public String getGroup() {
+ return group;
+ }
+
+ public void setGroup(String group) {
+ this.group = group;
+ }
+
+ public Set<NiFiUser> getUsers() {
+ return users;
+ }
+
+ public void setUsers(Set<NiFiUser> users) {
+ this.users = users;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/resources/nifi-administration-context.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/resources/nifi-administration-context.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/resources/nifi-administration-context.xml
new file mode 100644
index 0000000..a36619f
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/resources/nifi-administration-context.xml
@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<beans default-lazy-init="true"
+ xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns:context="http://www.springframework.org/schema/context"
+ xmlns:aop="http://www.springframework.org/schema/aop"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.1.xsd
+ http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-3.1.xsd
+ http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-3.1.xsd">
+
+ <!-- user authority provider -->
+ <bean id="authorityProvider" class="org.apache.nifi.authorization.AuthorityProviderFactoryBean" depends-on="clusterManager flowController">
+ <property name="properties" ref="nifiProperties"/>
+ </bean>
+
+ <!-- initialize the user data source -->
+ <bean id="userDataSource" class="org.apache.nifi.admin.UserDataSourceFactoryBean" destroy-method="shutdown">
+ <property name="properties" ref="nifiProperties"/>
+ </bean>
+
+ <!-- initialize the data source -->
+ <bean id="auditDataSource" class="org.apache.nifi.admin.AuditDataSourceFactoryBean" destroy-method="shutdown" depends-on="userDataSource">
+ <property name="properties" ref="nifiProperties"/>
+ </bean>
+
+ <!-- initialize the user transaction builder -->
+ <bean id="userTransactionBuilder" class="org.apache.nifi.admin.service.transaction.impl.StandardTransactionBuilder">
+ <property name="authorityProvider" ref="authorityProvider"/>
+ <property name="dataSource" ref="userDataSource"/>
+ </bean>
+
+ <!-- initialize the audit transaction builder -->
+ <bean id="auditTransactionBuilder" class="org.apache.nifi.admin.service.transaction.impl.StandardTransactionBuilder">
+ <property name="authorityProvider" ref="authorityProvider"/>
+ <property name="dataSource" ref="auditDataSource"/>
+ </bean>
+
+ <!-- administration service -->
+ <bean id="userService" class="org.apache.nifi.admin.service.impl.StandardUserService" init-method="seedUserAccounts">
+ <property name="transactionBuilder" ref="userTransactionBuilder"/>
+ <property name="properties" ref="nifiProperties"/>
+ </bean>
+
+ <!-- audit service -->
+ <bean id="auditService" class="org.apache.nifi.admin.service.impl.StandardAuditService">
+ <property name="transactionBuilder" ref="auditTransactionBuilder"/>
+ </bean>
+</beans>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/xsd/authority-providers.xsd
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/xsd/authority-providers.xsd b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/xsd/authority-providers.xsd
new file mode 100644
index 0000000..122fa2c
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/xsd/authority-providers.xsd
@@ -0,0 +1,49 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">
+ <!-- role -->
+ <xs:complexType name="Provider">
+ <xs:sequence>
+ <xs:element name="identifier" type="NonEmptyStringType"/>
+ <xs:element name="class" type="NonEmptyStringType"/>
+ <xs:element name="property" type="Property" minOccurs="0" maxOccurs="unbounded" />
+ </xs:sequence>
+ </xs:complexType>
+
+ <!-- Name/Value properties-->
+ <xs:complexType name="Property">
+ <xs:simpleContent>
+ <xs:extension base="xs:string">
+ <xs:attribute name="name" type="NonEmptyStringType"></xs:attribute>
+ </xs:extension>
+ </xs:simpleContent>
+ </xs:complexType>
+
+ <xs:simpleType name="NonEmptyStringType">
+ <xs:restriction base="xs:string">
+ <xs:minLength value="1"/>
+ </xs:restriction>
+ </xs:simpleType>
+
+ <!-- users -->
+ <xs:element name="authorityProviders">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="provider" type="Provider" minOccurs="0" maxOccurs="unbounded"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+</xs:schema>
\ No newline at end of file
[51/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
NIFI-850 removed nifi parent, updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/aa998847
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/aa998847
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/aa998847
Branch: refs/heads/develop
Commit: aa99884782e54c54ee138f5609b3be84628e96f9
Parents: 6fc52a4
Author: joewitt <jo...@apache.org>
Authored: Sat Aug 15 13:12:22 2015 -0400
Committer: joewitt <jo...@apache.org>
Committed: Sat Aug 15 13:12:22 2015 -0400
----------------------------------------------------------------------
LICENSE | 479 ++
NOTICE | 6 +
README.md | 155 +-
nifi-api/.gitignore | 2 +
nifi-api/pom.xml | 25 +
.../annotation/behavior/DynamicProperties.java | 37 +
.../annotation/behavior/DynamicProperty.java | 46 +
.../behavior/DynamicRelationship.java | 45 +
.../nifi/annotation/behavior/EventDriven.java | 48 +
.../annotation/behavior/ReadsAttribute.java | 41 +
.../annotation/behavior/ReadsAttributes.java | 39 +
.../annotation/behavior/SideEffectFree.java | 45 +
.../annotation/behavior/SupportsBatching.java | 51 +
.../annotation/behavior/TriggerSerially.java | 38 +
.../TriggerWhenAnyDestinationAvailable.java | 40 +
.../annotation/behavior/TriggerWhenEmpty.java | 41 +
.../annotation/behavior/WritesAttribute.java | 41 +
.../annotation/behavior/WritesAttributes.java | 39 +
.../documentation/CapabilityDescription.java | 41 +
.../nifi/annotation/documentation/SeeAlso.java | 44 +
.../nifi/annotation/documentation/Tags.java | 42 +
.../nifi/annotation/lifecycle/OnAdded.java | 52 +
.../nifi/annotation/lifecycle/OnDisabled.java | 62 +
.../nifi/annotation/lifecycle/OnEnabled.java | 69 +
.../nifi/annotation/lifecycle/OnRemoved.java | 55 +
.../nifi/annotation/lifecycle/OnScheduled.java | 67 +
.../nifi/annotation/lifecycle/OnShutdown.java | 53 +
.../nifi/annotation/lifecycle/OnStopped.java | 67 +
.../annotation/lifecycle/OnUnscheduled.java | 59 +
.../notification/OnPrimaryNodeStateChange.java | 44 +
.../notification/PrimaryNodeState.java | 33 +
.../apache/nifi/authorization/Authority.java | 93 +
.../nifi/authorization/AuthorityProvider.java | 182 +
.../AuthorityProviderConfigurationContext.java | 48 +
.../AuthorityProviderInitializationContext.java | 27 +
.../authorization/AuthorityProviderLookup.java | 25 +
.../authorization/DownloadAuthorization.java | 83 +
.../annotation/AuthorityProviderContext.java | 35 +
.../exception/AuthorityAccessException.java | 33 +
.../IdentityAlreadyExistsException.java | 32 +
.../exception/ProviderCreationException.java | 39 +
.../exception/ProviderDestructionException.java | 39 +
.../exception/UnknownIdentityException.java | 32 +
.../AbstractConfigurableComponent.java | 220 +
.../apache/nifi/components/AllowableValue.java | 128 +
.../nifi/components/ConfigurableComponent.java | 73 +
.../nifi/components/PropertyDescriptor.java | 556 +++
.../apache/nifi/components/PropertyValue.java | 189 +
.../nifi/components/ValidationContext.java | 95 +
.../nifi/components/ValidationResult.java | 172 +
.../org/apache/nifi/components/Validator.java | 54 +
.../controller/AbstractControllerService.java | 96 +
.../nifi/controller/ConfigurationContext.java | 61 +
.../nifi/controller/ControllerService.java | 180 +
.../ControllerServiceInitializationContext.java | 40 +
.../controller/ControllerServiceLookup.java | 70 +
.../apache/nifi/controller/FlowFileQueue.java | 203 +
.../apache/nifi/controller/ScheduledState.java | 37 +
.../org/apache/nifi/controller/Snippet.java | 89 +
.../org/apache/nifi/controller/Triggerable.java | 117 +
.../controller/annotation/OnConfigured.java | 44 +
.../repository/ContentRepository.java | 303 ++
.../controller/repository/FlowFileRecord.java | 48 +
.../repository/FlowFileRepository.java | 123 +
.../repository/FlowFileSwapManager.java | 63 +
.../controller/repository/QueueProvider.java | 33 +
.../controller/repository/RepositoryRecord.java | 82 +
.../repository/RepositoryRecordType.java | 25 +
.../repository/claim/ContentClaim.java | 52 +
.../repository/claim/ContentClaimManager.java | 142 +
.../controller/status/ConnectionStatus.java | 192 +
.../nifi/controller/status/PortStatus.java | 201 +
.../controller/status/ProcessGroupStatus.java | 584 +++
.../nifi/controller/status/ProcessorStatus.java | 274 ++
.../status/RemoteProcessGroupStatus.java | 217 +
.../nifi/controller/status/RunStatus.java | 28 +
.../controller/status/TransmissionStatus.java | 23 +
.../history/ComponentStatusRepository.java | 149 +
.../status/history/MetricDescriptor.java | 66 +
.../status/history/StatusHistory.java | 44 +
.../status/history/StatusSnapshot.java | 42 +
.../controller/status/history/ValueMapper.java | 23 +
.../controller/status/history/ValueReducer.java | 25 +
.../org/apache/nifi/events/EventReporter.java | 29 +
.../nifi/expression/AttributeExpression.java | 75 +
.../expression/AttributeValueDecorator.java | 29 +
.../expression/ExpressionLanguageCompiler.java | 69 +
.../java/org/apache/nifi/flowfile/FlowFile.java | 107 +
.../nifi/flowfile/FlowFilePrioritizer.java | 29 +
.../org/apache/nifi/logging/ComponentLog.java | 104 +
.../java/org/apache/nifi/logging/LogLevel.java | 27 +
.../org/apache/nifi/logging/ProcessorLog.java | 30 +
.../nifi/processor/AbstractProcessor.java | 37 +
.../AbstractSessionFactoryProcessor.java | 117 +
.../org/apache/nifi/processor/DataUnit.java | 245 +
.../apache/nifi/processor/FlowFileFilter.java | 93 +
.../apache/nifi/processor/ProcessContext.java | 129 +
.../apache/nifi/processor/ProcessSession.java | 736 +++
.../nifi/processor/ProcessSessionFactory.java | 30 +
.../org/apache/nifi/processor/Processor.java | 90 +
.../ProcessorInitializationContext.java | 47 +
.../org/apache/nifi/processor/QueueSize.java | 48 +
.../org/apache/nifi/processor/Relationship.java | 126 +
.../nifi/processor/SchedulingContext.java | 70 +
.../annotation/CapabilityDescription.java | 42 +
.../nifi/processor/annotation/EventDriven.java | 51 +
.../nifi/processor/annotation/OnAdded.java | 43 +
.../nifi/processor/annotation/OnRemoved.java | 44 +
.../nifi/processor/annotation/OnScheduled.java | 46 +
.../nifi/processor/annotation/OnShutdown.java | 40 +
.../nifi/processor/annotation/OnStopped.java | 58 +
.../processor/annotation/OnUnscheduled.java | 48 +
.../processor/annotation/SideEffectFree.java | 47 +
.../processor/annotation/SupportsBatching.java | 53 +
.../apache/nifi/processor/annotation/Tags.java | 43 +
.../processor/annotation/TriggerSerially.java | 41 +
.../TriggerWhenAnyDestinationAvailable.java | 43 +
.../processor/annotation/TriggerWhenEmpty.java | 43 +
.../exception/FlowFileAccessException.java | 36 +
.../exception/FlowFileHandlingException.java | 38 +
.../exception/MissingFlowFileException.java | 36 +
.../processor/exception/ProcessException.java | 43 +
.../nifi/processor/io/InputStreamCallback.java | 37 +
.../nifi/processor/io/OutputStreamCallback.java | 38 +
.../nifi/processor/io/StreamCallback.java | 37 +
.../nifi/provenance/ProvenanceEventBuilder.java | 308 ++
.../nifi/provenance/ProvenanceEventRecord.java | 235 +
.../provenance/ProvenanceEventRepository.java | 180 +
.../nifi/provenance/ProvenanceEventType.java | 92 +
.../nifi/provenance/ProvenanceReporter.java | 616 +++
.../lineage/ComputeLineageResult.java | 58 +
.../lineage/ComputeLineageSubmission.java | 70 +
.../apache/nifi/provenance/lineage/Lineage.java | 37 +
.../lineage/LineageComputationType.java | 27 +
.../nifi/provenance/lineage/LineageEdge.java | 26 +
.../nifi/provenance/lineage/LineageNode.java | 52 +
.../provenance/lineage/LineageNodeType.java | 23 +
.../lineage/ProvenanceEventLineageNode.java | 32 +
.../apache/nifi/provenance/search/Query.java | 96 +
.../nifi/provenance/search/QueryResult.java | 63 +
.../nifi/provenance/search/QuerySubmission.java | 55 +
.../nifi/provenance/search/SearchTerm.java | 24 +
.../nifi/provenance/search/SearchTerms.java | 39 +
.../nifi/provenance/search/SearchableField.java | 52 +
.../provenance/search/SearchableFieldType.java | 28 +
.../apache/nifi/remote/RemoteDestination.java | 50 +
.../nifi/reporting/AbstractReportingTask.java | 96 +
.../org/apache/nifi/reporting/Bulletin.java | 129 +
.../apache/nifi/reporting/BulletinQuery.java | 121 +
.../nifi/reporting/BulletinRepository.java | 82 +
.../apache/nifi/reporting/ComponentType.java | 58 +
.../org/apache/nifi/reporting/EventAccess.java | 49 +
.../nifi/reporting/InitializationException.java | 32 +
.../apache/nifi/reporting/ReportingContext.java | 89 +
.../ReportingInitializationContext.java | 77 +
.../apache/nifi/reporting/ReportingTask.java | 75 +
.../org/apache/nifi/reporting/Severity.java | 24 +
.../nifi/scheduling/SchedulingStrategy.java | 86 +
.../org/apache/nifi/search/SearchContext.java | 49 +
.../org/apache/nifi/search/SearchResult.java | 78 +
.../java/org/apache/nifi/search/Searchable.java | 27 +
.../nifi/web/ClusterRequestException.java | 39 +
.../org/apache/nifi/web/ComponentDetails.java | 144 +
.../apache/nifi/web/ConfigurationAction.java | 125 +
.../nifi/web/InvalidRevisionException.java | 33 +
.../nifi/web/NiFiWebConfigurationContext.java | 100 +
.../web/NiFiWebConfigurationRequestContext.java | 32 +
.../org/apache/nifi/web/NiFiWebContext.java | 116 +
.../apache/nifi/web/NiFiWebContextConfig.java | 55 +
.../apache/nifi/web/NiFiWebRequestContext.java | 56 +
.../nifi/web/ProcessorConfigurationAction.java | 126 +
.../java/org/apache/nifi/web/ProcessorInfo.java | 111 +
.../nifi/web/ResourceNotFoundException.java | 32 +
.../main/java/org/apache/nifi/web/Revision.java | 81 +
.../org/apache/nifi/web/UiExtensionType.java | 32 +
.../org/apache/nifi/web/ViewableContent.java | 65 +
.../nifi/components/TestPropertyDescriptor.java | 59 +
.../org/apache/nifi/processor/TestDataUnit.java | 44 +
nifi-api/src/test/resources/logback-test.xml | 27 +
nifi-assembly/LICENSE | 915 ++++
nifi-assembly/NOTICE | 788 ++++
nifi-assembly/README.md | 110 +
nifi-assembly/pom.xml | 501 +++
.../src/main/assembly/dependencies.xml | 140 +
nifi-bootstrap/pom.xml | 32 +
.../apache/nifi/bootstrap/BootstrapCodec.java | 92 +
.../org/apache/nifi/bootstrap/NiFiListener.java | 141 +
.../java/org/apache/nifi/bootstrap/RunNiFi.java | 1008 +++++
.../org/apache/nifi/bootstrap/ShutdownHook.java | 92 +
.../exception/InvalidCommandException.java | 38 +
.../bootstrap/util/LimitingInputStream.java | 107 +
.../nifi-data-provenance-utils/.gitignore | 2 +
nifi-commons/nifi-data-provenance-utils/pom.xml | 34 +
.../nifi/provenance/AsyncLineageSubmission.java | 88 +
.../nifi/provenance/AsyncQuerySubmission.java | 78 +
.../nifi/provenance/NamedSearchableField.java | 95 +
.../nifi/provenance/SearchableFieldParser.java | 53 +
.../nifi/provenance/SearchableFields.java | 91 +
.../nifi/provenance/StandardLineageResult.java | 326 ++
.../StandardProvenanceEventRecord.java | 751 ++++
.../nifi/provenance/StandardQueryResult.java | 168 +
.../nifi/provenance/lineage/EdgeNode.java | 74 +
.../nifi/provenance/lineage/EventNode.java | 111 +
.../provenance/lineage/FlowFileLineage.java | 76 +
.../nifi/provenance/lineage/FlowFileNode.java | 84 +
nifi-commons/nifi-expression-language/pom.xml | 56 +
.../language/antlr/AttributeExpressionLexer.g | 211 +
.../language/antlr/AttributeExpressionParser.g | 140 +
.../output/AttributeExpressionLexer.tokens | 88 +
.../expression/language/EmptyPreparedQuery.java | 62 +
.../language/InvalidPreparedQuery.java | 72 +
.../expression/language/PreparedQuery.java | 39 +
.../attribute/expression/language/Query.java | 1313 ++++++
.../language/StandardAttributeExpression.java | 65 +
.../StandardExpressionLanguageCompiler.java | 58 +
.../language/StandardPreparedQuery.java | 83 +
.../language/evaluation/BooleanEvaluator.java | 43 +
.../language/evaluation/BooleanQueryResult.java | 43 +
.../language/evaluation/DateEvaluator.java | 45 +
.../language/evaluation/DateQueryResult.java | 45 +
.../language/evaluation/Evaluator.java | 44 +
.../language/evaluation/NumberEvaluator.java | 43 +
.../language/evaluation/NumberQueryResult.java | 43 +
.../language/evaluation/QueryResult.java | 26 +
.../language/evaluation/StringEvaluator.java | 43 +
.../language/evaluation/StringQueryResult.java | 43 +
.../evaluation/cast/BooleanCastEvaluator.java | 50 +
.../evaluation/cast/DateCastEvaluator.java | 117 +
.../evaluation/cast/NumberCastEvaluator.java | 72 +
.../evaluation/cast/StringCastEvaluator.java | 49 +
.../evaluation/functions/AndEvaluator.java | 60 +
.../evaluation/functions/AppendEvaluator.java | 50 +
.../functions/AttributeEvaluator.java | 45 +
.../evaluation/functions/ContainsEvaluator.java | 52 +
.../functions/DateToNumberEvaluator.java | 50 +
.../evaluation/functions/DivideEvaluator.java | 57 +
.../evaluation/functions/EndsWithEvaluator.java | 52 +
.../evaluation/functions/EqualsEvaluator.java | 89 +
.../functions/EqualsIgnoreCaseEvaluator.java | 60 +
.../evaluation/functions/FindEvaluator.java | 71 +
.../evaluation/functions/FormatEvaluator.java | 61 +
.../functions/GreaterThanEvaluator.java | 56 +
.../functions/GreaterThanOrEqualEvaluator.java | 56 +
.../evaluation/functions/HostnameEvaluator.java | 58 +
.../evaluation/functions/IPEvaluator.java | 46 +
.../evaluation/functions/IndexOfEvaluator.java | 52 +
.../evaluation/functions/IsEmptyEvaluator.java | 44 +
.../evaluation/functions/IsNullEvaluator.java | 45 +
.../functions/LastIndexOfEvaluator.java | 52 +
.../evaluation/functions/LengthEvaluator.java | 45 +
.../evaluation/functions/LessThanEvaluator.java | 56 +
.../functions/LessThanOrEqualEvaluator.java | 56 +
.../evaluation/functions/MatchesEvaluator.java | 70 +
.../evaluation/functions/MinusEvaluator.java | 57 +
.../evaluation/functions/ModEvaluator.java | 57 +
.../evaluation/functions/MultiplyEvaluator.java | 57 +
.../evaluation/functions/NotEvaluator.java | 49 +
.../evaluation/functions/NotNullEvaluator.java | 45 +
.../evaluation/functions/NowEvaluator.java | 39 +
.../functions/NumberToDateEvaluator.java | 51 +
.../functions/OneUpSequenceEvaluator.java | 41 +
.../evaluation/functions/OrEvaluator.java | 60 +
.../evaluation/functions/PlusEvaluator.java | 57 +
.../evaluation/functions/PrependEvaluator.java | 50 +
.../functions/ReplaceAllEvaluator.java | 55 +
.../functions/ReplaceEmptyEvaluator.java | 51 +
.../evaluation/functions/ReplaceEvaluator.java | 55 +
.../functions/ReplaceNullEvaluator.java | 47 +
.../functions/StartsWithEvaluator.java | 52 +
.../functions/StringToDateEvaluator.java | 64 +
.../functions/SubstringAfterEvaluator.java | 59 +
.../functions/SubstringAfterLastEvaluator.java | 55 +
.../functions/SubstringBeforeEvaluator.java | 58 +
.../functions/SubstringBeforeLastEvaluator.java | 55 +
.../functions/SubstringEvaluator.java | 64 +
.../evaluation/functions/ToLowerEvaluator.java | 45 +
.../evaluation/functions/ToNumberEvaluator.java | 45 +
.../evaluation/functions/ToRadixEvaluator.java | 76 +
.../evaluation/functions/ToStringEvaluator.java | 45 +
.../evaluation/functions/ToUpperEvaluator.java | 45 +
.../evaluation/functions/TrimEvaluator.java | 45 +
.../functions/UrlDecodeEvaluator.java | 55 +
.../functions/UrlEncodeEvaluator.java | 55 +
.../evaluation/functions/UuidEvaluator.java | 39 +
.../literals/BooleanLiteralEvaluator.java | 44 +
.../literals/NumberLiteralEvaluator.java | 44 +
.../literals/StringLiteralEvaluator.java | 77 +
.../evaluation/literals/ToLiteralEvaluator.java | 43 +
.../evaluation/reduce/CountEvaluator.java | 56 +
.../evaluation/reduce/JoinEvaluator.java | 60 +
.../evaluation/reduce/ReduceEvaluator.java | 23 +
.../selection/AllAttributesEvaluator.java | 73 +
.../selection/AnyAttributeEvaluator.java | 73 +
.../selection/DelineatedAttributeEvaluator.java | 87 +
.../selection/IteratingEvaluator.java | 33 +
.../evaluation/selection/MappingEvaluator.java | 71 +
.../selection/MultiAttributeEvaluator.java | 23 +
.../selection/MultiMatchAttributeEvaluator.java | 87 +
.../selection/MultiNamedAttributeEvaluator.java | 69 +
.../AttributeExpressionLanguageException.java | 34 +
...ibuteExpressionLanguageParsingException.java | 34 +
.../exception/IllegalAttributeException.java | 30 +
.../expression/language/TestQuery.java | 1176 +++++
.../language/TestStandardPreparedQuery.java | 92 +
nifi-commons/nifi-flowfile-packager/pom.xml | 34 +
.../org/apache/nifi/util/FlowFilePackager.java | 28 +
.../apache/nifi/util/FlowFilePackagerV1.java | 104 +
.../apache/nifi/util/FlowFilePackagerV2.java | 146 +
.../apache/nifi/util/FlowFilePackagerV3.java | 93 +
.../apache/nifi/util/FlowFileUnpackager.java | 30 +
.../apache/nifi/util/FlowFileUnpackagerV1.java | 155 +
.../apache/nifi/util/FlowFileUnpackagerV2.java | 143 +
.../apache/nifi/util/FlowFileUnpackagerV3.java | 161 +
.../java/org/apache/nifi/util/Unpackage.java | 119 +
.../nifi/util/TestPackageUnpackageV3.java | 56 +
nifi-commons/nifi-hl7-query-language/.gitignore | 3 +
nifi-commons/nifi-hl7-query-language/pom.xml | 127 +
.../apache/nifi/hl7/query/antlr/HL7QueryLexer.g | 173 +
.../nifi/hl7/query/antlr/HL7QueryParser.g | 108 +
.../org/apache/nifi/hl7/hapi/EmptyField.java | 37 +
.../org/apache/nifi/hl7/hapi/HapiField.java | 84 +
.../org/apache/nifi/hl7/hapi/HapiMessage.java | 95 +
.../org/apache/nifi/hl7/hapi/HapiSegment.java | 69 +
.../apache/nifi/hl7/hapi/SingleValueField.java | 43 +
.../java/org/apache/nifi/hl7/io/HL7Reader.java | 27 +
.../hl7/io/exception/InvalidHL7Exception.java | 41 +
.../org/apache/nifi/hl7/model/HL7Component.java | 26 +
.../org/apache/nifi/hl7/model/HL7Field.java | 20 +
.../org/apache/nifi/hl7/model/HL7Message.java | 27 +
.../org/apache/nifi/hl7/model/HL7Segment.java | 27 +
.../org/apache/nifi/hl7/query/Declaration.java | 29 +
.../org/apache/nifi/hl7/query/HL7Query.java | 384 ++
.../org/apache/nifi/hl7/query/QueryResult.java | 30 +
.../org/apache/nifi/hl7/query/ResultHit.java | 26 +
.../org/apache/nifi/hl7/query/Selection.java | 38 +
.../hl7/query/evaluator/BooleanEvaluator.java | 24 +
.../nifi/hl7/query/evaluator/Evaluator.java | 28 +
.../hl7/query/evaluator/IntegerEvaluator.java | 25 +
.../hl7/query/evaluator/StringEvaluator.java | 25 +
.../comparison/AbstractComparisonEvaluator.java | 111 +
.../comparison/AbstractNumericComparison.java | 68 +
.../evaluator/comparison/EqualsEvaluator.java | 32 +
.../comparison/GreaterThanEvaluator.java | 32 +
.../comparison/GreaterThanOrEqualEvaluator.java | 32 +
.../evaluator/comparison/IsNullEvaluator.java | 70 +
.../evaluator/comparison/LessThanEvaluator.java | 32 +
.../comparison/LessThanOrEqualEvaluator.java | 32 +
.../comparison/NotEqualsEvaluator.java | 32 +
.../evaluator/comparison/NotEvaluator.java | 37 +
.../evaluator/comparison/NotNullEvaluator.java | 66 +
.../literal/IntegerLiteralEvaluator.java | 36 +
.../literal/StringLiteralEvaluator.java | 36 +
.../hl7/query/evaluator/logic/AndEvaluator.java | 44 +
.../hl7/query/evaluator/logic/OrEvaluator.java | 44 +
.../message/DeclaredReferenceEvaluator.java | 43 +
.../query/evaluator/message/DotEvaluator.java | 89 +
.../query/evaluator/message/FieldEvaluator.java | 68 +
.../evaluator/message/MessageEvaluator.java | 34 +
.../evaluator/message/SegmentEvaluator.java | 52 +
.../exception/HL7QueryParsingException.java | 38 +
.../nifi/hl7/query/result/MissedResult.java | 57 +
.../hl7/query/result/StandardQueryResult.java | 70 +
.../hl7/query/result/StandardResultHit.java | 42 +
.../org/apache/nifi/hl7/query/TestHL7Query.java | 306 ++
.../src/test/resources/hyperglycemia | 5 +
.../src/test/resources/hypoglycemia | 5 +
nifi-commons/nifi-logging-utils/pom.xml | 31 +
.../java/org/apache/nifi/logging/NiFiLog.java | 366 ++
nifi-commons/nifi-processor-utilities/pom.xml | 40 +
.../nifi/processor/util/FlowFileFilters.java | 65 +
.../nifi/processor/util/SSLProperties.java | 236 +
.../nifi/processor/util/StandardValidators.java | 708 +++
.../apache/nifi/processor/TestFormatUtils.java | 40 +
.../processor/util/TestStandardValidators.java | 89 +
nifi-commons/nifi-properties/.gitignore | 3 +
nifi-commons/nifi-properties/pom.xml | 24 +
.../org/apache/nifi/util/NiFiProperties.java | 932 ++++
.../java/org/apache/nifi/util/StringUtils.java | 66 +
.../apache/nifi/util/NiFiPropertiesTest.java | 127 +
.../NiFiProperties/conf/nifi.blank.properties | 128 +
.../NiFiProperties/conf/nifi.missing.properties | 126 +
.../NiFiProperties/conf/nifi.properties | 128 +
nifi-commons/nifi-security-utils/pom.xml | 35 +
.../nifi/security/util/CertificateUtils.java | 159 +
.../nifi/security/util/EncryptionMethod.java | 86 +
.../apache/nifi/security/util/KeystoreType.java | 26 +
.../nifi/security/util/SecurityStoreTypes.java | 144 +
.../nifi/security/util/SslContextFactory.java | 182 +
nifi-commons/nifi-site-to-site-client/pom.xml | 54 +
.../remote/AbstractCommunicationsSession.java | 55 +
.../org/apache/nifi/remote/Communicant.java | 46 +
.../main/java/org/apache/nifi/remote/Peer.java | 151 +
.../org/apache/nifi/remote/PeerDescription.java | 80 +
.../java/org/apache/nifi/remote/PeerStatus.java | 61 +
.../nifi/remote/RemoteResourceInitiator.java | 75 +
.../org/apache/nifi/remote/Transaction.java | 225 +
.../nifi/remote/TransactionCompletion.java | 64 +
.../apache/nifi/remote/TransferDirection.java | 33 +
.../nifi/remote/VersionedRemoteResource.java | 24 +
.../apache/nifi/remote/client/KeystoreType.java | 24 +
.../nifi/remote/client/SiteToSiteClient.java | 771 ++++
.../remote/client/SiteToSiteClientConfig.java | 148 +
.../client/socket/EndpointConnection.java | 55 +
.../client/socket/EndpointConnectionPool.java | 994 +++++
.../nifi/remote/client/socket/SocketClient.java | 224 +
.../remote/cluster/AdaptedNodeInformation.java | 66 +
.../remote/cluster/ClusterNodeInformation.java | 67 +
.../nifi/remote/cluster/NodeInformation.java | 98 +
.../remote/cluster/NodeInformationAdapter.java | 41 +
.../apache/nifi/remote/codec/FlowFileCodec.java | 72 +
.../remote/codec/StandardFlowFileCodec.java | 128 +
.../remote/exception/HandshakeException.java | 38 +
.../exception/PortNotRunningException.java | 31 +
.../remote/exception/ProtocolException.java | 40 +
.../remote/exception/UnknownPortException.java | 31 +
.../SocketChannelCommunicationsSession.java | 111 +
.../remote/io/socket/SocketChannelInput.java | 72 +
.../remote/io/socket/SocketChannelOutput.java | 59 +
.../SSLSocketChannelCommunicationsSession.java | 114 +
.../io/socket/ssl/SSLSocketChannelInput.java | 56 +
.../io/socket/ssl/SSLSocketChannelOutput.java | 45 +
.../nifi/remote/protocol/ClientProtocol.java | 76 +
.../remote/protocol/CommunicationsInput.java | 34 +
.../remote/protocol/CommunicationsOutput.java | 27 +
.../remote/protocol/CommunicationsSession.java | 62 +
.../apache/nifi/remote/protocol/DataPacket.java | 48 +
.../nifi/remote/protocol/RequestType.java | 43 +
.../protocol/socket/HandshakeProperty.java | 59 +
.../nifi/remote/protocol/socket/Response.java | 52 +
.../remote/protocol/socket/ResponseCode.java | 148 +
.../protocol/socket/SocketClientProtocol.java | 446 ++
.../socket/SocketClientTransaction.java | 406 ++
.../SocketClientTransactionCompletion.java | 57 +
.../nifi/remote/util/NiFiRestApiUtil.java | 100 +
.../nifi/remote/util/PeerStatusCache.java | 44 +
.../nifi/remote/util/StandardDataPacket.java | 50 +
.../socket/TestEndpointConnectionStatePool.java | 92 +
.../client/socket/TestSiteToSiteClient.java | 103 +
nifi-commons/nifi-socket-utils/pom.xml | 55 +
.../nifi/io/nio/AbstractChannelReader.java | 162 +
.../java/org/apache/nifi/io/nio/BufferPool.java | 110 +
.../apache/nifi/io/nio/ChannelDispatcher.java | 158 +
.../org/apache/nifi/io/nio/ChannelListener.java | 227 +
.../nifi/io/nio/DatagramChannelReader.java | 59 +
.../apache/nifi/io/nio/SocketChannelReader.java | 51 +
.../io/nio/consumer/AbstractStreamConsumer.java | 131 +
.../nifi/io/nio/consumer/StreamConsumer.java | 80 +
.../io/nio/consumer/StreamConsumerFactory.java | 26 +
.../nifi/io/socket/SSLContextFactory.java | 102 +
.../io/socket/ServerSocketConfiguration.java | 81 +
.../nifi/io/socket/SocketConfiguration.java | 114 +
.../apache/nifi/io/socket/SocketListener.java | 210 +
.../org/apache/nifi/io/socket/SocketUtils.java | 167 +
.../socket/multicast/DiscoverableService.java | 42 +
.../multicast/DiscoverableServiceImpl.java | 77 +
.../multicast/MulticastConfiguration.java | 98 +
.../io/socket/multicast/MulticastListener.java | 192 +
.../multicast/MulticastServiceDiscovery.java | 33 +
.../multicast/MulticastServicesBroadcaster.java | 32 +
.../socket/multicast/MulticastTimeToLive.java | 49 +
.../io/socket/multicast/MulticastUtils.java | 108 +
.../io/socket/multicast/ServiceDiscovery.java | 30 +
.../socket/multicast/ServicesBroadcaster.java | 55 +
.../apache/nifi/io/nio/example/ServerMain.java | 140 +
.../apache/nifi/io/nio/example/TCPClient.java | 86 +
.../apache/nifi/io/nio/example/UDPClient.java | 50 +
.../io/nio/example/UselessStreamConsumer.java | 42 +
.../src/test/resources/log4j.xml | 36 +
nifi-commons/nifi-utils/.gitignore | 8 +
nifi-commons/nifi-utils/pom.xml | 30 +
.../flowfile/attributes/CoreAttributes.java | 65 +
.../attributes/FlowFileAttributeKey.java | 22 +
.../nifi/remote/StandardVersionNegotiator.java | 81 +
.../apache/nifi/remote/VersionNegotiator.java | 60 +
.../TransmissionDisabledException.java | 24 +
.../nifi/remote/io/CompressionInputStream.java | 185 +
.../nifi/remote/io/CompressionOutputStream.java | 146 +
.../remote/io/InterruptableInputStream.java | 117 +
.../remote/io/InterruptableOutputStream.java | 81 +
.../remote/io/socket/BufferStateManager.java | 109 +
.../io/socket/SocketChannelInputStream.java | 170 +
.../io/socket/SocketChannelOutputStream.java | 114 +
.../remote/io/socket/ssl/SSLSocketChannel.java | 614 +++
.../socket/ssl/SSLSocketChannelInputStream.java | 65 +
.../ssl/SSLSocketChannelOutputStream.java | 52 +
.../nifi/stream/io/BufferedInputStream.java | 34 +
.../nifi/stream/io/BufferedOutputStream.java | 129 +
.../nifi/stream/io/ByteArrayInputStream.java | 205 +
.../nifi/stream/io/ByteArrayOutputStream.java | 220 +
.../nifi/stream/io/ByteCountingInputStream.java | 110 +
.../stream/io/ByteCountingOutputStream.java | 66 +
.../apache/nifi/stream/io/DataOutputStream.java | 370 ++
.../apache/nifi/stream/io/GZIPOutputStream.java | 39 +
.../stream/io/LeakyBucketStreamThrottler.java | 330 ++
.../nifi/stream/io/LimitingInputStream.java | 111 +
.../stream/io/MinimumLengthInputStream.java | 91 +
.../nifi/stream/io/NonCloseableInputStream.java | 55 +
.../stream/io/NonCloseableOutputStream.java | 51 +
.../apache/nifi/stream/io/NullOutputStream.java | 46 +
.../apache/nifi/stream/io/StreamThrottler.java | 33 +
.../org/apache/nifi/stream/io/StreamUtils.java | 245 +
.../apache/nifi/stream/io/ZipOutputStream.java | 37 +
.../exception/BytePatternNotFoundException.java | 28 +
.../io/util/NonThreadSafeCircularBuffer.java | 69 +
.../org/apache/nifi/util/BooleanHolder.java | 25 +
.../java/org/apache/nifi/util/EscapeUtils.java | 41 +
.../java/org/apache/nifi/util/FormatUtils.java | 205 +
.../org/apache/nifi/util/IntegerHolder.java | 54 +
.../java/org/apache/nifi/util/LongHolder.java | 58 +
.../apache/nifi/util/NaiveSearchRingBuffer.java | 122 +
.../java/org/apache/nifi/util/ObjectHolder.java | 38 +
.../java/org/apache/nifi/util/RingBuffer.java | 281 ++
.../java/org/apache/nifi/util/StopWatch.java | 126 +
.../main/java/org/apache/nifi/util/Tuple.java | 77 +
.../concurrency/DebugDisabledTimedLock.java | 64 +
.../util/concurrency/DebugEnabledTimedLock.java | 130 +
.../util/concurrency/DebuggableTimedLock.java | 30 +
.../apache/nifi/util/concurrency/TimedLock.java | 59 +
.../org/apache/nifi/util/file/FileUtils.java | 594 +++
.../file/monitor/CompoundUpdateMonitor.java | 112 +
.../util/file/monitor/LastModifiedMonitor.java | 30 +
.../nifi/util/file/monitor/MD5SumMonitor.java | 51 +
.../file/monitor/SynchronousFileWatcher.java | 121 +
.../nifi/util/file/monitor/UpdateMonitor.java | 25 +
.../org/apache/nifi/util/search/Search.java | 49 +
.../org/apache/nifi/util/search/SearchTerm.java | 138 +
.../util/search/ahocorasick/AhoCorasick.java | 155 +
.../nifi/util/search/ahocorasick/Node.java | 71 +
.../util/search/ahocorasick/SearchState.java | 63 +
.../nifi/util/timebuffer/EntityAccess.java | 26 +
.../nifi/util/timebuffer/LongEntityAccess.java | 43 +
.../nifi/util/timebuffer/TimedBuffer.java | 114 +
.../nifi/util/timebuffer/TimestampedLong.java | 35 +
.../io/TestCompressionInputOutputStreams.java | 153 +
.../stream/io/ByteCountingInputStreamTest.java | 55 +
.../stream/io/TestLeakyBucketThrottler.java | 147 +
.../nifi/util/TestNaiveSearchRingBuffer.java | 72 +
.../file/monitor/TestCompoundUpdateMonitor.java | 70 +
.../monitor/TestSynchronousFileWatcher.java | 60 +
.../nifi/util/timebuffer/TestRingBuffer.java | 192 +
.../nifi/util/timebuffer/TestTimedBuffer.java | 106 +
.../src/test/resources/logback-test.xml | 32 +
nifi-commons/nifi-web-utils/pom.xml | 51 +
.../org/apache/nifi/web/util/ClientUtils.java | 132 +
.../nifi/web/util/ObjectMapperResolver.java | 48 +
.../java/org/apache/nifi/web/util/WebUtils.java | 197 +
nifi-commons/nifi-write-ahead-log/.gitignore | 2 +
nifi-commons/nifi-write-ahead-log/pom.xml | 35 +
.../org/wali/MinimalLockingWriteAheadLog.java | 1028 +++++
.../src/main/java/org/wali/SerDe.java | 128 +
.../src/main/java/org/wali/SyncListener.java | 62 +
.../src/main/java/org/wali/UpdateType.java | 49 +
.../java/org/wali/WriteAheadRepository.java | 122 +
.../src/test/java/org/wali/DummyRecord.java | 66 +
.../test/java/org/wali/DummyRecordSerde.java | 115 +
.../wali/TestMinimalLockingWriteAheadLog.java | 513 +++
nifi-commons/pom.xml | 41 +
nifi-docs/LICENSE | 235 +
nifi-docs/NOTICE | 5 +
nifi-docs/pom.xml | 152 +
.../src/main/asciidoc/administration-guide.adoc | 692 +++
nifi-docs/src/main/asciidoc/asciidoc-mod.css | 418 ++
.../src/main/asciidoc/developer-guide.adoc | 2170 +++++++++
.../asciidoc/expression-language-guide.adoc | 1745 ++++++++
.../images/add-controller-service-window.png | Bin 0 -> 157241 bytes
.../images/add-processor-with-tag-cloud.png | Bin 0 -> 22925 bytes
.../src/main/asciidoc/images/add-processor.png | Bin 0 -> 31524 bytes
.../src/main/asciidoc/images/addConnect.png | Bin 0 -> 1996 bytes
.../src/main/asciidoc/images/comments-tab.png | Bin 0 -> 6431 bytes
.../src/main/asciidoc/images/components.png | Bin 0 -> 8992 bytes
.../configure-controller-service-properties.png | Bin 0 -> 107830 bytes
.../configure-controller-service-settings.png | Bin 0 -> 79718 bytes
.../configure-reporting-task-properties.png | Bin 0 -> 58292 bytes
.../configure-reporting-task-settings.png | Bin 0 -> 80213 bytes
.../asciidoc/images/connection-settings.png | Bin 0 -> 15214 bytes
.../images/controller-services-edit-buttons.png | Bin 0 -> 22294 bytes
.../controller-services-edit-buttons2.png | Bin 0 -> 21845 bytes
.../asciidoc/images/controller-services-tab.png | Bin 0 -> 167278 bytes
.../images/controller-settings-button.png | Bin 0 -> 32341 bytes
.../main/asciidoc/images/create-connection.png | Bin 0 -> 9430 bytes
.../images/create-service-ssl-context.png | Bin 0 -> 142764 bytes
.../asciidoc/images/edit-property-dropdown.png | Bin 0 -> 14291 bytes
.../asciidoc/images/edit-property-textarea.png | Bin 0 -> 17146 bytes
.../main/asciidoc/images/event-attributes.png | Bin 0 -> 94668 bytes
.../src/main/asciidoc/images/event-content.png | Bin 0 -> 88726 bytes
.../src/main/asciidoc/images/event-details.png | Bin 0 -> 139736 bytes
.../src/main/asciidoc/images/expand-event.png | Bin 0 -> 40476 bytes
.../main/asciidoc/images/expanded-events.png | Bin 0 -> 76082 bytes
.../src/main/asciidoc/images/find-parents.png | Bin 0 -> 35831 bytes
.../src/main/asciidoc/images/iconAlert.png | Bin 0 -> 1396 bytes
.../src/main/asciidoc/images/iconConnection.png | Bin 0 -> 1517 bytes
.../src/main/asciidoc/images/iconDelete.png | Bin 0 -> 670 bytes
.../src/main/asciidoc/images/iconDisable.png | Bin 0 -> 764 bytes
nifi-docs/src/main/asciidoc/images/iconEdit.png | Bin 0 -> 493 bytes
.../src/main/asciidoc/images/iconEnable.png | Bin 0 -> 667 bytes
.../src/main/asciidoc/images/iconExport.png | Bin 0 -> 453 bytes
.../main/asciidoc/images/iconFlowHistory.png | Bin 0 -> 2463 bytes
.../src/main/asciidoc/images/iconFunnel.png | Bin 0 -> 1223 bytes
nifi-docs/src/main/asciidoc/images/iconInfo.png | Bin 0 -> 550 bytes
.../src/main/asciidoc/images/iconInputPort.png | Bin 0 -> 1842 bytes
.../main/asciidoc/images/iconInputPortSmall.png | Bin 0 -> 532 bytes
.../src/main/asciidoc/images/iconLabel.png | Bin 0 -> 838 bytes
.../src/main/asciidoc/images/iconLineage.png | Bin 0 -> 2214 bytes
.../main/asciidoc/images/iconNewTemplate.png | Bin 0 -> 557 bytes
.../src/main/asciidoc/images/iconNotSecure.png | Bin 0 -> 221 bytes
.../src/main/asciidoc/images/iconOutputPort.png | Bin 0 -> 1658 bytes
.../asciidoc/images/iconOutputPortSmall.png | Bin 0 -> 459 bytes
.../main/asciidoc/images/iconProcessGroup.png | Bin 0 -> 1422 bytes
.../src/main/asciidoc/images/iconProcessor.png | Bin 0 -> 1446 bytes
.../src/main/asciidoc/images/iconProvenance.png | Bin 0 -> 2268 bytes
.../asciidoc/images/iconRemoteProcessGroup.png | Bin 0 -> 674 bytes
.../src/main/asciidoc/images/iconResize.png | Bin 0 -> 165 bytes
nifi-docs/src/main/asciidoc/images/iconRun.png | Bin 0 -> 538 bytes
.../src/main/asciidoc/images/iconSecure.png | Bin 0 -> 225 bytes
.../src/main/asciidoc/images/iconSettings.png | Bin 0 -> 2638 bytes
nifi-docs/src/main/asciidoc/images/iconStop.png | Bin 0 -> 402 bytes
.../src/main/asciidoc/images/iconSummary.png | Bin 0 -> 272 bytes
.../src/main/asciidoc/images/iconTemplate.png | Bin 0 -> 970 bytes
.../asciidoc/images/iconTransmissionActive.png | Bin 0 -> 1330 bytes
.../images/iconTransmissionInactive.png | Bin 0 -> 1248 bytes
.../src/main/asciidoc/images/iconUsers.png | Bin 0 -> 2272 bytes
.../main/asciidoc/images/iconViewDetails.png | Bin 0 -> 1788 bytes
.../images/instantiate-template-description.png | Bin 0 -> 6508 bytes
.../asciidoc/images/instantiate-template.png | Bin 0 -> 3216 bytes
.../main/asciidoc/images/invalid-processor.png | Bin 0 -> 8187 bytes
.../main/asciidoc/images/lineage-flowfile.png | Bin 0 -> 3855 bytes
.../asciidoc/images/lineage-graph-annotated.png | Bin 0 -> 170122 bytes
nifi-docs/src/main/asciidoc/images/ncm.png | Bin 0 -> 339522 bytes
nifi-docs/src/main/asciidoc/images/new-flow.png | Bin 0 -> 266913 bytes
.../main/asciidoc/images/nifi-arch-cluster.png | Bin 0 -> 50384 bytes
.../src/main/asciidoc/images/nifi-arch.png | Bin 0 -> 43509 bytes
.../main/asciidoc/images/nifi-navigation.png | Bin 0 -> 339195 bytes
.../asciidoc/images/nifi-toolbar-components.png | Bin 0 -> 262314 bytes
.../images/nifi_first_launch_screenshot.png | Bin 0 -> 122540 bytes
.../src/main/asciidoc/images/parent-found.png | Bin 0 -> 42814 bytes
.../asciidoc/images/process-group-anatomy.png | Bin 0 -> 48554 bytes
.../main/asciidoc/images/processor-anatomy.png | Bin 0 -> 37035 bytes
.../images/processor-connection-bubble.png | Bin 0 -> 9936 bytes
.../src/main/asciidoc/images/properties-tab.png | Bin 0 -> 11847 bytes
.../asciidoc/images/provenance-annotated.png | Bin 0 -> 530929 bytes
.../main/asciidoc/images/provenance-table.png | Bin 0 -> 466599 bytes
.../asciidoc/images/remote-group-anatomy.png | Bin 0 -> 61383 bytes
.../images/remote-group-ports-dialog.png | Bin 0 -> 21662 bytes
.../images/remote-port-connection-status.png | Bin 0 -> 57332 bytes
.../images/reporting-tasks-edit-buttons.png | Bin 0 -> 5049 bytes
.../images/reporting-tasks-edit-buttons2.png | Bin 0 -> 17525 bytes
.../asciidoc/images/reporting-tasks-tab.png | Bin 0 -> 161551 bytes
.../src/main/asciidoc/images/scheduling-tab.png | Bin 0 -> 12288 bytes
.../src/main/asciidoc/images/search-events.png | Bin 0 -> 62626 bytes
.../images/search-receive-event-abc.png | Bin 0 -> 67817 bytes
.../asciidoc/images/settings-general-tab.png | Bin 0 -> 132657 bytes
.../src/main/asciidoc/images/settings-tab.png | Bin 0 -> 16225 bytes
.../src/main/asciidoc/images/simple-flow.png | Bin 0 -> 196808 bytes
.../src/main/asciidoc/images/stats-history.png | Bin 0 -> 33205 bytes
.../src/main/asciidoc/images/status-bar.png | Bin 0 -> 249077 bytes
.../main/asciidoc/images/summary-annotated.png | Bin 0 -> 111956 bytes
.../src/main/asciidoc/images/summary-table.png | Bin 0 -> 62114 bytes
.../main/asciidoc/images/valid-processor.png | Bin 0 -> 8090 bytes
nifi-docs/src/main/asciidoc/overview.adoc | 296 ++
nifi-docs/src/main/asciidoc/user-guide.adoc | 1468 ++++++
nifi-docs/src/main/assembly/dependencies.xml | 44 +
nifi-external/README.md | 19 +
.../nifi-nifi-example-nar/pom.xml | 35 +
.../nifi-nifi-example-processors/pom.xml | 70 +
.../nifi/processors/WriteResourceToStream.java | 102 +
.../org.apache.nifi.processor.Processor | 15 +
.../src/main/resources/file.txt | 1 +
.../processors/WriteResourceToStreamTest.java | 47 +
nifi-external/nifi-example-bundle/pom.xml | 42 +
nifi-external/nifi-spark-receiver/pom.xml | 36 +
.../org/apache/nifi/spark/NiFiDataPacket.java | 39 +
.../org/apache/nifi/spark/NiFiReceiver.java | 208 +
nifi-external/nifi-storm-spout/pom.xml | 38 +
.../org/apache/nifi/storm/NiFiDataPacket.java | 39 +
.../java/org/apache/nifi/storm/NiFiSpout.java | 266 ++
nifi-external/pom.xml | 32 +
.../nifi-processor-bundle-archetype/pom.xml | 47 +
.../META-INF/maven/archetype-metadata.xml | 43 +
.../nifi-__artifactBaseName__-nar/pom.xml | 37 +
.../pom.xml | 54 +
.../src/main/java/MyProcessor.java | 96 +
.../org.apache.nifi.processor.Processor | 15 +
.../src/test/java/MyProcessorTest.java | 39 +
.../main/resources/archetype-resources/pom.xml | 35 +
.../nifi-service-bundle-archetype/pom.xml | 48 +
.../META-INF/maven/archetype-metadata.xml | 55 +
.../nifi-__artifactBaseName__-api-nar/pom.xml | 42 +
.../nifi-__artifactBaseName__-api/pom.xml | 35 +
.../src/main/java/MyService.java | 30 +
.../nifi-__artifactBaseName__-nar/pom.xml | 43 +
.../nifi-__artifactBaseName__/pom.xml | 59 +
.../src/main/java/StandardMyService.java | 80 +
...org.apache.nifi.controller.ControllerService | 15 +
.../src/test/java/TestProcessor.java | 45 +
.../src/test/java/TestStandardMyService.java | 45 +
.../main/resources/archetype-resources/pom.xml | 37 +
nifi-maven-archetypes/pom.xml | 30 +
nifi-mock/pom.xml | 49 +
.../MockProvenanceEventRepository.java | 131 +
.../apache/nifi/reporting/BulletinFactory.java | 43 +
.../org/apache/nifi/reporting/MockBulletin.java | 24 +
.../util/ControllerServiceConfiguration.java | 74 +
.../nifi/util/MockBulletinRepository.java | 74 +
.../nifi/util/MockConfigurationContext.java | 77 +
...kControllerServiceInitializationContext.java | 58 +
.../nifi/util/MockControllerServiceLookup.java | 100 +
.../org/apache/nifi/util/MockEventAccess.java | 70 +
.../java/org/apache/nifi/util/MockFlowFile.java | 277 ++
.../org/apache/nifi/util/MockFlowFileQueue.java | 85 +
.../apache/nifi/util/MockProcessContext.java | 286 ++
.../apache/nifi/util/MockProcessSession.java | 1037 +++++
.../MockProcessorInitializationContext.java | 84 +
.../org/apache/nifi/util/MockProcessorLog.java | 276 ++
.../org/apache/nifi/util/MockPropertyValue.java | 185 +
.../nifi/util/MockProvenanceReporter.java | 453 ++
.../apache/nifi/util/MockReportingContext.java | 115 +
.../MockReportingInitializationContext.java | 89 +
.../apache/nifi/util/MockSessionFactory.java | 48 +
.../apache/nifi/util/MockValidationContext.java | 135 +
.../org/apache/nifi/util/ReflectionUtils.java | 150 +
.../apache/nifi/util/SharedSessionState.java | 91 +
.../apache/nifi/util/SingleSessionFactory.java | 35 +
.../nifi/util/StandardProcessorTestRunner.java | 728 +++
.../java/org/apache/nifi/util/TestRunner.java | 765 ++++
.../java/org/apache/nifi/util/TestRunners.java | 37 +
.../CurrentTestStandardProcessorTestRunner.java | 71 +
.../util/TestStandardProcessorTestRunner.java | 90 +
.../nifi-aws-bundle/nifi-aws-nar/pom.xml | 36 +
.../src/main/resources/META-INF/NOTICE | 74 +
.../nifi-aws-bundle/nifi-aws-processors/pom.xml | 67 +
.../processors/aws/AbstractAWSProcessor.java | 190 +
.../processors/aws/s3/AbstractS3Processor.java | 169 +
.../nifi/processors/aws/s3/FetchS3Object.java | 155 +
.../nifi/processors/aws/s3/PutS3Object.java | 181 +
.../aws/sns/AbstractSNSProcessor.java | 58 +
.../apache/nifi/processors/aws/sns/PutSNS.java | 155 +
.../aws/sqs/AbstractSQSProcessor.java | 51 +
.../nifi/processors/aws/sqs/DeleteSQS.java | 95 +
.../apache/nifi/processors/aws/sqs/GetSQS.java | 205 +
.../apache/nifi/processors/aws/sqs/PutSQS.java | 146 +
.../org.apache.nifi.processor.Processor | 20 +
.../processors/aws/s3/TestFetchS3Object.java | 61 +
.../nifi/processors/aws/s3/TestPutS3Object.java | 102 +
.../nifi/processors/aws/sns/TestPutSNS.java | 51 +
.../nifi/processors/aws/sqs/TestGetSQS.java | 49 +
.../nifi/processors/aws/sqs/TestPutSQS.java | 52 +
.../src/test/resources/hello.txt | 1 +
nifi-nar-bundles/nifi-aws-bundle/pom.xml | 43 +
.../nifi-flume-bundle/nifi-flume-nar/pom.xml | 132 +
.../src/main/resources/META-INF/LICENSE | 319 ++
.../src/main/resources/META-INF/NOTICE | 192 +
.../nifi-flume-processors/pom.xml | 178 +
.../flume/AbstractFlumeProcessor.java | 160 +
.../nifi/processors/flume/ExecuteFlumeSink.java | 144 +
.../processors/flume/ExecuteFlumeSource.java | 195 +
.../processors/flume/NifiChannelSelector.java | 69 +
.../processors/flume/NifiSessionChannel.java | 47 +
.../flume/NifiSessionFactoryChannel.java | 50 +
.../flume/NifiSinkSessionChannel.java | 49 +
.../processors/flume/NifiSinkTransaction.java | 71 +
.../nifi/processors/flume/NifiTransaction.java | 54 +
.../processors/flume/util/FlowFileEvent.java | 134 +
.../flume/util/FlowFileEventConstants.java | 40 +
.../org.apache.nifi.processor.Processor | 16 +
.../additionalDetails.html | 159 +
.../additionalDetails.html | 114 +
.../processors/flume/ExecuteFlumeSinkTest.java | 169 +
.../flume/ExecuteFlumeSourceTest.java | 151 +
.../src/test/resources/core-site-broken.xml | 25 +
.../src/test/resources/core-site.xml | 30 +
.../src/test/resources/log4j.properties | 22 +
.../src/test/resources/simplelogger.properties | 21 +
.../src/test/resources/testdata/records.txt | 4 +
nifi-nar-bundles/nifi-flume-bundle/pom.xml | 40 +
.../nifi-framework-nar/.gitignore | 1 +
.../nifi-framework-nar/pom.xml | 62 +
.../src/main/resources/META-INF/LICENSE | 264 ++
.../src/main/resources/META-INF/NOTICE | 137 +
.../nifi-administration/.gitignore | 1 +
.../nifi-framework/nifi-administration/pom.xml | 121 +
.../nifi/admin/AuditDataSourceFactoryBean.java | 222 +
.../org/apache/nifi/admin/RepositoryUtils.java | 90 +
.../nifi/admin/UserDataSourceFactoryBean.java | 235 +
.../org/apache/nifi/admin/dao/ActionDAO.java | 72 +
.../org/apache/nifi/admin/dao/AuthorityDAO.java | 59 +
.../org/apache/nifi/admin/dao/DAOFactory.java | 29 +
.../nifi/admin/dao/DataAccessException.java | 39 +
.../java/org/apache/nifi/admin/dao/UserDAO.java | 127 +
.../nifi/admin/dao/impl/DAOFactoryImpl.java | 51 +
.../nifi/admin/dao/impl/StandardActionDAO.java | 965 ++++
.../admin/dao/impl/StandardAuthorityDAO.java | 172 +
.../nifi/admin/dao/impl/StandardUserDAO.java | 634 +++
.../admin/service/AccountDisabledException.java | 40 +
.../admin/service/AccountNotFoundException.java | 40 +
.../admin/service/AccountPendingException.java | 41 +
.../admin/service/AdministrationException.java | 39 +
.../apache/nifi/admin/service/AuditService.java | 74 +
.../apache/nifi/admin/service/UserService.java | 157 +
.../service/action/AbstractUserAction.java | 97 +
.../admin/service/action/AddActionsAction.java | 48 +
.../service/action/AdministrationAction.java | 38 +
.../service/action/AuthorizeDownloadAction.java | 54 +
.../service/action/AuthorizeUserAction.java | 173 +
.../admin/service/action/CreateUserAction.java | 53 +
.../admin/service/action/DeleteUserAction.java | 68 +
.../admin/service/action/DisableUserAction.java | 76 +
.../service/action/DisableUserGroupAction.java | 69 +
.../service/action/FindUserByDnAction.java | 49 +
.../service/action/FindUserByIdAction.java | 46 +
.../admin/service/action/GetActionAction.java | 41 +
.../admin/service/action/GetActionsAction.java | 48 +
.../admin/service/action/GetPreviousValues.java | 43 +
.../service/action/GetUserGroupAction.java | 50 +
.../admin/service/action/GetUsersAction.java | 39 +
.../service/action/HasPendingUserAccounts.java | 34 +
.../action/InvalidateUserAccountAction.java | 58 +
.../InvalidateUserGroupAccountsAction.java | 45 +
.../service/action/PurgeActionsAction.java | 51 +
.../action/RequestUserAccountAction.java | 67 +
.../service/action/SeedUserAccountsAction.java | 164 +
.../admin/service/action/UngroupUserAction.java | 69 +
.../service/action/UngroupUserGroupAction.java | 57 +
.../admin/service/action/UpdateUserAction.java | 124 +
.../UpdateUserAuthoritiesCacheAction.java | 73 +
.../service/action/UpdateUserCacheAction.java | 47 +
.../service/action/UpdateUserGroupAction.java | 171 +
.../service/impl/StandardAuditService.java | 219 +
.../admin/service/impl/StandardUserService.java | 642 +++
.../admin/service/transaction/Transaction.java | 49 +
.../service/transaction/TransactionBuilder.java | 25 +
.../transaction/TransactionException.java | 40 +
.../transaction/impl/StandardTransaction.java | 93 +
.../impl/StandardTransactionBuilder.java | 57 +
.../AuthorityProviderFactoryBean.java | 490 ++
...rdAuthorityProviderConfigurationContext.java | 50 +
...dAuthorityProviderInitializationContext.java | 42 +
.../java/org/apache/nifi/history/History.java | 56 +
.../org/apache/nifi/history/HistoryQuery.java | 99 +
.../org/apache/nifi/history/PreviousValue.java | 54 +
.../org/apache/nifi/user/AccountStatus.java | 47 +
.../java/org/apache/nifi/user/NiFiUser.java | 164 +
.../org/apache/nifi/user/NiFiUserGroup.java | 45 +
.../resources/nifi-administration-context.xml | 62 +
.../src/main/xsd/authority-providers.xsd | 49 +
.../service/action/AuthorizeUserActionTest.java | 433 ++
.../service/action/CreateUserActionTest.java | 144 +
.../service/action/DisableUserActionTest.java | 168 +
.../action/InvalidateUserAccountActionTest.java | 126 +
.../action/RequestUserAccountActionTest.java | 127 +
.../action/SeedUserAccountsActionTest.java | 262 ++
.../action/SetUserAuthoritiesActionTest.java | 223 +
.../nifi-framework/nifi-client-dto/.gitignore | 6 +
.../nifi-framework/nifi-client-dto/pom.xml | 30 +
.../org/apache/nifi/web/api/dto/AboutDTO.java | 64 +
.../org/apache/nifi/web/api/dto/BannerDTO.java | 64 +
.../nifi/web/api/dto/BulletinBoardDTO.java | 65 +
.../apache/nifi/web/api/dto/BulletinDTO.java | 168 +
.../nifi/web/api/dto/BulletinQueryDTO.java | 120 +
.../org/apache/nifi/web/api/dto/ClusterDTO.java | 64 +
.../nifi/web/api/dto/ComponentHistoryDTO.java | 59 +
.../apache/nifi/web/api/dto/ConnectableDTO.java | 157 +
.../apache/nifi/web/api/dto/ConnectionDTO.java | 237 +
.../web/api/dto/ControllerConfigurationDTO.java | 187 +
.../apache/nifi/web/api/dto/ControllerDTO.java | 277 ++
.../nifi/web/api/dto/ControllerServiceDTO.java | 204 +
...ontrollerServiceReferencingComponentDTO.java | 219 +
.../org/apache/nifi/web/api/dto/CounterDTO.java | 101 +
.../apache/nifi/web/api/dto/CountersDTO.java | 64 +
.../nifi/web/api/dto/DocumentedTypeDTO.java | 75 +
.../apache/nifi/web/api/dto/FlowSnippetDTO.java | 165 +
.../org/apache/nifi/web/api/dto/FunnelDTO.java | 29 +
.../org/apache/nifi/web/api/dto/LabelDTO.java | 100 +
.../nifi/web/api/dto/NiFiComponentDTO.java | 106 +
.../org/apache/nifi/web/api/dto/NodeDTO.java | 202 +
.../apache/nifi/web/api/dto/NodeEventDTO.java | 78 +
.../web/api/dto/NodeSystemDiagnosticsDTO.java | 59 +
.../org/apache/nifi/web/api/dto/PortDTO.java | 173 +
.../apache/nifi/web/api/dto/PositionDTO.java | 68 +
.../nifi/web/api/dto/PreviousValueDTO.java | 77 +
.../nifi/web/api/dto/ProcessGroupDTO.java | 236 +
.../nifi/web/api/dto/ProcessorConfigDTO.java | 292 ++
.../apache/nifi/web/api/dto/ProcessorDTO.java | 199 +
.../nifi/web/api/dto/PropertyDescriptorDTO.java | 251 ++
.../nifi/web/api/dto/PropertyHistoryDTO.java | 45 +
.../nifi/web/api/dto/RelationshipDTO.java | 73 +
.../api/dto/RemoteProcessGroupContentsDTO.java | 59 +
.../nifi/web/api/dto/RemoteProcessGroupDTO.java | 291 ++
.../web/api/dto/RemoteProcessGroupPortDTO.java | 202 +
.../nifi/web/api/dto/ReportingTaskDTO.java | 253 ++
.../apache/nifi/web/api/dto/RevisionDTO.java | 82 +
.../org/apache/nifi/web/api/dto/SnippetDTO.java | 247 +
.../nifi/web/api/dto/SystemDiagnosticsDTO.java | 491 ++
.../apache/nifi/web/api/dto/TemplateDTO.java | 124 +
.../org/apache/nifi/web/api/dto/UserDTO.java | 187 +
.../apache/nifi/web/api/dto/UserGroupDTO.java | 89 +
.../nifi/web/api/dto/action/ActionDTO.java | 187 +
.../nifi/web/api/dto/action/HistoryDTO.java | 78 +
.../web/api/dto/action/HistoryQueryDTO.java | 153 +
.../component/details/ComponentDetailsDTO.java | 32 +
.../component/details/ExtensionDetailsDTO.java | 43 +
.../details/RemoteProcessGroupDetailsDTO.java | 43 +
.../dto/action/details/ActionDetailsDTO.java | 34 +
.../dto/action/details/ConfigureDetailsDTO.java | 73 +
.../dto/action/details/ConnectDetailsDTO.java | 134 +
.../api/dto/action/details/MoveDetailsDTO.java | 88 +
.../api/dto/action/details/PurgeDetailsDTO.java | 47 +
.../web/api/dto/provenance/AttributeDTO.java | 73 +
.../web/api/dto/provenance/ProvenanceDTO.java | 174 +
.../api/dto/provenance/ProvenanceEventDTO.java | 662 +++
.../dto/provenance/ProvenanceOptionsDTO.java | 45 +
.../dto/provenance/ProvenanceRequestDTO.java | 125 +
.../dto/provenance/ProvenanceResultsDTO.java | 144 +
.../ProvenanceSearchableFieldDTO.java | 89 +
.../api/dto/provenance/lineage/LineageDTO.java | 171 +
.../provenance/lineage/LineageRequestDTO.java | 90 +
.../provenance/lineage/LineageResultsDTO.java | 77 +
.../provenance/lineage/ProvenanceLinkDTO.java | 107 +
.../provenance/lineage/ProvenanceNodeDTO.java | 172 +
.../dto/search/ComponentSearchResultDTO.java | 90 +
.../web/api/dto/search/NodeSearchResultDTO.java | 59 +
.../web/api/dto/search/SearchResultsDTO.java | 136 +
.../dto/search/UserGroupSearchResultDTO.java | 44 +
.../web/api/dto/search/UserSearchResultDTO.java | 59 +
.../dto/status/ClusterConnectionStatusDTO.java | 94 +
.../api/dto/status/ClusterPortStatusDTO.java | 94 +
.../status/ClusterProcessGroupStatusDTO.java | 102 +
.../dto/status/ClusterProcessorStatusDTO.java | 127 +
.../ClusterRemoteProcessGroupStatusDTO.java | 94 +
.../web/api/dto/status/ClusterStatusDTO.java | 46 +
.../api/dto/status/ClusterStatusHistoryDTO.java | 79 +
.../web/api/dto/status/ConnectionStatusDTO.java | 211 +
.../web/api/dto/status/ControllerStatusDTO.java | 231 +
.../api/dto/status/NodeConnectionStatusDTO.java | 60 +
.../web/api/dto/status/NodePortStatusDTO.java | 60 +
.../dto/status/NodeProcessGroupStatusDTO.java | 64 +
.../api/dto/status/NodeProcessorStatusDTO.java | 60 +
.../status/NodeRemoteProcessGroupStatusDTO.java | 60 +
.../nifi/web/api/dto/status/NodeStatusDTO.java | 60 +
.../api/dto/status/NodeStatusHistoryDTO.java | 60 +
.../nifi/web/api/dto/status/PortStatusDTO.java | 149 +
.../api/dto/status/ProcessGroupStatusDTO.java | 371 ++
.../web/api/dto/status/ProcessorStatusDTO.java | 214 +
.../web/api/dto/status/RemotePortStatusDTO.java | 104 +
.../dto/status/RemoteProcessGroupStatusDTO.java | 167 +
.../nifi/web/api/dto/status/StatusDTO.java | 45 +
.../web/api/dto/status/StatusDescriptorDTO.java | 105 +
.../web/api/dto/status/StatusHistoryDTO.java | 97 +
.../api/dto/status/StatusHistoryDetailDTO.java | 59 +
.../web/api/dto/status/StatusSnapshotDTO.java | 61 +
.../nifi/web/api/dto/util/DateTimeAdapter.java | 47 +
.../nifi/web/api/dto/util/TimeAdapter.java | 47 +
.../nifi/web/api/dto/util/TimestampAdapter.java | 47 +
.../apache/nifi/web/api/entity/AboutEntity.java | 43 +
.../nifi/web/api/entity/ActionEntity.java | 43 +
.../nifi/web/api/entity/AuthorityEntity.java | 54 +
.../nifi/web/api/entity/BannerEntity.java | 44 +
.../web/api/entity/BulletinBoardEntity.java | 43 +
.../entity/ClusterConnectionStatusEntity.java | 43 +
.../nifi/web/api/entity/ClusterEntity.java | 43 +
.../web/api/entity/ClusterPortStatusEntity.java | 43 +
.../entity/ClusterProcessGroupStatusEntity.java | 43 +
.../entity/ClusterProcessorStatusEntity.java | 43 +
.../ClusterRemoteProcessGroupStatusEntity.java | 44 +
.../api/entity/ClusterSearchResultsEntity.java | 42 +
.../web/api/entity/ClusterStatusEntity.java | 43 +
.../api/entity/ClusterStatusHistoryEntity.java | 43 +
.../web/api/entity/ComponentHistoryEntity.java | 43 +
.../nifi/web/api/entity/ConnectionEntity.java | 41 +
.../nifi/web/api/entity/ConnectionsEntity.java | 43 +
.../entity/ControllerConfigurationEntity.java | 43 +
.../nifi/web/api/entity/ControllerEntity.java | 43 +
.../web/api/entity/ControllerServiceEntity.java | 41 +
...ollerServiceReferencingComponentsEntity.java | 43 +
.../entity/ControllerServiceTypesEntity.java | 42 +
.../api/entity/ControllerServicesEntity.java | 42 +
.../web/api/entity/ControllerStatusEntity.java | 43 +
.../nifi/web/api/entity/CounterEntity.java | 41 +
.../nifi/web/api/entity/CountersEntity.java | 41 +
.../org/apache/nifi/web/api/entity/Entity.java | 45 +
.../nifi/web/api/entity/FlowSnippetEntity.java | 43 +
.../nifi/web/api/entity/FunnelEntity.java | 43 +
.../nifi/web/api/entity/FunnelsEntity.java | 43 +
.../nifi/web/api/entity/HistoryEntity.java | 43 +
.../nifi/web/api/entity/InputPortEntity.java | 41 +
.../nifi/web/api/entity/InputPortsEntity.java | 43 +
.../apache/nifi/web/api/entity/LabelEntity.java | 43 +
.../nifi/web/api/entity/LabelsEntity.java | 43 +
.../nifi/web/api/entity/LineageEntity.java | 43 +
.../apache/nifi/web/api/entity/NodeEntity.java | 43 +
.../nifi/web/api/entity/NodeStatusEntity.java | 43 +
.../api/entity/NodeSystemDiagnosticsEntity.java | 43 +
.../nifi/web/api/entity/OutputPortEntity.java | 41 +
.../nifi/web/api/entity/OutputPortsEntity.java | 43 +
.../web/api/entity/PrioritizerTypesEntity.java | 42 +
.../nifi/web/api/entity/ProcessGroupEntity.java | 43 +
.../api/entity/ProcessGroupStatusEntity.java | 43 +
.../web/api/entity/ProcessGroupsEntity.java | 44 +
.../nifi/web/api/entity/ProcessorEntity.java | 43 +
.../web/api/entity/ProcessorTypesEntity.java | 42 +
.../nifi/web/api/entity/ProcessorsEntity.java | 43 +
.../api/entity/PropertyDescriptorEntity.java | 43 +
.../nifi/web/api/entity/ProvenanceEntity.java | 38 +
.../web/api/entity/ProvenanceEventEntity.java | 43 +
.../web/api/entity/ProvenanceOptionsEntity.java | 44 +
.../api/entity/RemoteProcessGroupEntity.java | 43 +
.../entity/RemoteProcessGroupPortEntity.java | 43 +
.../api/entity/RemoteProcessGroupsEntity.java | 43 +
.../web/api/entity/ReportingTaskEntity.java | 41 +
.../api/entity/ReportingTaskTypesEntity.java | 42 +
.../web/api/entity/ReportingTasksEntity.java | 42 +
.../web/api/entity/SearchResultsEntity.java | 42 +
.../nifi/web/api/entity/SnippetEntity.java | 43 +
.../web/api/entity/StatusHistoryEntity.java | 43 +
.../web/api/entity/SystemDiagnosticsEntity.java | 43 +
.../nifi/web/api/entity/TemplateEntity.java | 43 +
.../nifi/web/api/entity/TemplatesEntity.java | 59 +
.../apache/nifi/web/api/entity/UserEntity.java | 43 +
.../nifi/web/api/entity/UserGroupEntity.java | 43 +
.../web/api/entity/UserSearchResultsEntity.java | 56 +
.../apache/nifi/web/api/entity/UsersEntity.java | 60 +
.../.gitignore | 1 +
.../nifi-cluster-authorization-provider/pom.xml | 46 +
.../ClusterManagerAuthorizationProvider.java | 225 +
.../NodeAuthorizationProvider.java | 389 ++
.../protocol/message/DoesDnExistMessage.java | 55 +
.../protocol/message/GetAuthoritiesMessage.java | 57 +
.../message/GetGroupForUserMessage.java | 54 +
.../protocol/message/ProtocolMessage.java | 56 +
.../message/jaxb/JaxbProtocolUtils.java | 41 +
.../protocol/message/jaxb/ObjectFactory.java | 44 +
....apache.nifi.authorization.AuthorityProvider | 16 +
.../nifi-framework/nifi-documentation/pom.xml | 44 +
.../ConfigurableComponentInitializer.java | 45 +
.../apache/nifi/documentation/DocGenerator.java | 180 +
.../nifi/documentation/DocumentationWriter.java | 33 +
.../html/HtmlDocumentationWriter.java | 614 +++
.../html/HtmlProcessorDocumentationWriter.java | 256 ++
.../init/ControllerServiceInitializer.java | 57 +
.../init/ProcessorInitializer.java | 55 +
.../init/ReportingTaskingInitializer.java | 54 +
.../mock/MockConfigurationContext.java | 48 +
...kControllerServiceInitializationContext.java | 46 +
.../mock/MockControllerServiceLookup.java | 65 +
.../documentation/mock/MockProcessContext.java | 85 +
.../MockProcessorInitializationContext.java | 45 +
.../documentation/mock/MockProcessorLogger.java | 169 +
.../MockReportingInitializationContext.java | 67 +
.../documentation/util/ReflectionUtils.java | 139 +
.../nifi/documentation/DocGeneratorTest.java | 96 +
.../example/ControllerServiceWithLogger.java | 31 +
.../FullyDocumentedControllerService.java | 103 +
.../example/FullyDocumentedProcessor.java | 172 +
.../example/FullyDocumentedReportingTask.java | 94 +
.../documentation/example/NakedProcessor.java | 31 +
.../example/ProcessorWithLogger.java | 37 +
.../example/ReportingTaskWithLogger.java | 36 +
.../documentation/example/SampleService.java | 25 +
.../html/HtmlDocumentationWriterTest.java | 156 +
.../html/ProcessorDocumentationWriterTest.java | 132 +
.../nifi/documentation/html/XmlValidator.java | 53 +
.../src/test/resources/conf/nifi.properties | 129 +
.../src/test/resources/lib/example.nar | Bin 0 -> 721040 bytes
.../src/test/resources/lib/jetty.nar | Bin 0 -> 4638519 bytes
.../test/resources/lib/nifi-framework-nar.nar | Bin 0 -> 406 bytes
.../nifi-file-authorization-provider/pom.xml | 85 +
.../FileAuthorizationProvider.java | 496 +++
....apache.nifi.authorization.AuthorityProvider | 15 +
.../src/main/xsd/users.xsd | 64 +
.../FileAuthorizationProviderTest.java | 128 +
.../nifi-framework-cluster-protocol/.gitignore | 1 +
.../nifi-framework-cluster-protocol/pom.xml | 67 +
.../protocol/ClusterManagerProtocolSender.java | 73 +
.../cluster/protocol/ConnectionRequest.java | 43 +
.../cluster/protocol/ConnectionResponse.java | 139 +
.../apache/nifi/cluster/protocol/Heartbeat.java | 68 +
.../nifi/cluster/protocol/NodeBulletins.java | 44 +
.../nifi/cluster/protocol/NodeIdentifier.java | 183 +
.../cluster/protocol/NodeProtocolSender.java | 84 +
.../nifi/cluster/protocol/ProtocolContext.java | 40 +
.../cluster/protocol/ProtocolException.java | 40 +
.../nifi/cluster/protocol/ProtocolHandler.java | 45 +
.../nifi/cluster/protocol/ProtocolListener.java | 77 +
.../protocol/ProtocolMessageMarshaller.java | 38 +
.../protocol/ProtocolMessageUnmarshaller.java | 39 +
.../nifi/cluster/protocol/StandardDataFlow.java | 98 +
.../UnknownServiceAddressException.java | 39 +
.../impl/ClusterManagerProtocolSenderImpl.java | 246 +
.../ClusterManagerProtocolSenderListener.java | 117 +
.../protocol/impl/ClusterServiceDiscovery.java | 180 +
.../protocol/impl/ClusterServiceLocator.java | 228 +
.../impl/ClusterServicesBroadcaster.java | 185 +
.../protocol/impl/CopyingInputStream.java | 78 +
.../impl/MulticastProtocolListener.java | 203 +
.../protocol/impl/NodeProtocolSenderImpl.java | 170 +
.../impl/NodeProtocolSenderListener.java | 115 +
.../protocol/impl/SocketProtocolListener.java | 204 +
.../protocol/jaxb/JaxbProtocolContext.java | 147 +
.../jaxb/message/AdaptedConnectionRequest.java | 40 +
.../jaxb/message/AdaptedConnectionResponse.java | 109 +
.../protocol/jaxb/message/AdaptedCounter.java | 56 +
.../protocol/jaxb/message/AdaptedDataFlow.java | 64 +
.../protocol/jaxb/message/AdaptedHeartbeat.java | 66 +
.../jaxb/message/AdaptedNodeBulletins.java | 50 +
.../jaxb/message/AdaptedNodeIdentifier.java | 76 +
.../jaxb/message/ConnectionRequestAdapter.java | 40 +
.../jaxb/message/ConnectionResponseAdapter.java | 54 +
.../protocol/jaxb/message/DataFlowAdapter.java | 49 +
.../protocol/jaxb/message/HeartbeatAdapter.java | 53 +
.../jaxb/message/JaxbProtocolUtils.java | 41 +
.../jaxb/message/NodeBulletinsAdapter.java | 47 +
.../jaxb/message/NodeIdentifierAdapter.java | 50 +
.../protocol/jaxb/message/ObjectFactory.java | 104 +
.../message/ConnectionRequestMessage.java | 46 +
.../message/ConnectionResponseMessage.java | 63 +
.../ControllerStartupFailureMessage.java | 49 +
.../protocol/message/DisconnectMessage.java | 54 +
.../protocol/message/ExceptionMessage.java | 44 +
.../protocol/message/FlowRequestMessage.java | 45 +
.../protocol/message/FlowResponseMessage.java | 43 +
.../protocol/message/HeartbeatMessage.java | 42 +
.../message/MulticastProtocolMessage.java | 66 +
.../protocol/message/NodeBulletinsMessage.java | 42 +
.../cluster/protocol/message/PingMessage.java | 55 +
.../message/PrimaryRoleAssignmentMessage.java | 55 +
.../protocol/message/ProtocolMessage.java | 61 +
.../message/ReconnectionFailureMessage.java | 47 +
.../message/ReconnectionRequestMessage.java | 94 +
.../message/ReconnectionResponseMessage.java | 32 +
.../message/ServiceBroadcastMessage.java | 64 +
.../MulticastConfigurationFactoryBean.java | 60 +
.../ServerSocketConfigurationFactoryBean.java | 66 +
.../spring/SocketConfigurationFactoryBean.java | 66 +
.../resources/nifi-cluster-protocol-context.xml | 110 +
.../ClusterManagerProtocolSenderImplTest.java | 133 +
.../impl/ClusterServiceDiscoveryTest.java | 134 +
.../impl/ClusterServiceLocatorTest.java | 121 +
.../impl/ClusterServicesBroadcasterTest.java | 131 +
.../impl/MulticastProtocolListenerTest.java | 168 +
.../impl/NodeProtocolSenderImplTest.java | 198 +
.../impl/SocketProtocolListenerTest.java | 132 +
.../impl/testutils/DelayedProtocolHandler.java | 56 +
.../testutils/ReflexiveProtocolHandler.java | 46 +
.../nifi-framework-cluster-web/.gitignore | 1 +
.../nifi-framework-cluster-web/pom.xml | 44 +
.../nifi/cluster/context/ClusterContext.java | 65 +
.../cluster/context/ClusterContextImpl.java | 69 +
.../context/ClusterContextThreadLocal.java | 42 +
.../nifi-framework-cluster/.gitignore | 1 +
.../nifi-framework-cluster/pom.xml | 152 +
.../cluster/client/MulticastTestClient.java | 151 +
.../org/apache/nifi/cluster/event/Event.java | 119 +
.../apache/nifi/cluster/event/EventManager.java | 62 +
.../cluster/event/impl/EventManagerImpl.java | 140 +
.../cluster/firewall/ClusterNodeFirewall.java | 34 +
.../impl/FileBasedClusterNodeFirewall.java | 207 +
.../nifi/cluster/flow/ClusterDataFlow.java | 55 +
.../apache/nifi/cluster/flow/DaoException.java | 39 +
.../apache/nifi/cluster/flow/DataFlowDao.java | 60 +
.../cluster/flow/DataFlowManagementService.java | 124 +
.../nifi/cluster/flow/PersistedFlowState.java | 36 +
.../nifi/cluster/flow/StaleFlowException.java | 40 +
.../nifi/cluster/flow/impl/DataFlowDaoImpl.java | 597 +++
.../impl/DataFlowManagementServiceImpl.java | 403 ++
.../nifi/cluster/manager/ClusterManager.java | 182 +
.../cluster/manager/HttpClusterManager.java | 129 +
.../cluster/manager/HttpRequestReplicator.java | 83 +
.../cluster/manager/HttpResponseMapper.java | 40 +
.../nifi/cluster/manager/NodeResponse.java | 321 ++
.../exception/BlockedByFirewallException.java | 60 +
.../manager/exception/ClusterException.java | 39 +
.../ConnectingNodeMutableRequestException.java | 39 +
...DisconnectedNodeMutableRequestException.java | 39 +
.../exception/IllegalClusterStateException.java | 39 +
.../exception/IllegalNodeDeletionException.java | 39 +
.../IllegalNodeDisconnectionException.java | 39 +
.../IllegalNodeReconnectionException.java | 39 +
.../IneligiblePrimaryNodeException.java | 39 +
.../exception/MutableRequestException.java | 40 +
.../exception/NoConnectedNodesException.java | 39 +
.../exception/NoResponseFromNodesException.java | 40 +
.../exception/NodeDisconnectionException.java | 39 +
.../exception/NodeReconnectionException.java | 39 +
.../PrimaryRoleAssignmentException.java | 39 +
.../SafeModeMutableRequestException.java | 39 +
.../manager/exception/UnknownNodeException.java | 39 +
.../exception/UriConstructionException.java | 39 +
.../manager/impl/ClusteredEventAccess.java | 135 +
.../manager/impl/ClusteredReportingContext.java | 209 +
.../manager/impl/HttpRequestReplicatorImpl.java | 515 +++
.../manager/impl/HttpResponseMapperImpl.java | 81 +
.../cluster/manager/impl/WebClusterManager.java | 4212 ++++++++++++++++++
.../java/org/apache/nifi/cluster/node/Node.java | 237 +
...anagerProtocolServiceLocatorFactoryBean.java | 114 +
...FileBasedClusterNodeFirewallFactoryBean.java | 58 +
.../spring/WebClusterManagerFactoryBean.java | 132 +
.../reporting/ClusteredReportingTaskNode.java | 49 +
.../resources/nifi-cluster-manager-context.xml | 128 +
.../event/impl/EventManagerImplTest.java | 119 +
.../impl/FileBasedClusterNodeFirewallTest.java | 135 +
.../impl/DataFlowManagementServiceImplTest.java | 345 ++
.../impl/HttpRequestReplicatorImplTest.java | 368 ++
.../impl/HttpResponseMapperImplTest.java | 125 +
.../manager/impl/TestWebClusterManager.java | 54 +
.../cluster/manager/testutils/HttpRequest.java | 235 +
.../cluster/manager/testutils/HttpResponse.java | 91 +
.../manager/testutils/HttpResponseAction.java | 57 +
.../cluster/manager/testutils/HttpServer.java | 238 +
.../src/test/resources/logback-test.xml | 48 +
.../apache/nifi/cluster/firewall/impl/empty.txt | 0
.../apache/nifi/cluster/firewall/impl/ips.txt | 12 +
.../nifi-framework-core-api/.gitignore | 2 +
.../nifi-framework-core-api/pom.xml | 54 +
.../apache/nifi/cluster/protocol/DataFlow.java | 41 +
.../apache/nifi/connectable/Connectable.java | 274 ++
.../nifi/connectable/ConnectableType.java | 44 +
.../org/apache/nifi/connectable/Connection.java | 78 +
.../org/apache/nifi/connectable/Funnel.java | 24 +
.../java/org/apache/nifi/connectable/Port.java | 31 +
.../org/apache/nifi/connectable/Position.java | 36 +
.../java/org/apache/nifi/connectable/Size.java | 36 +
.../controller/AbstractConfiguredComponent.java | 295 ++
.../apache/nifi/controller/AbstractPort.java | 608 +++
.../nifi/controller/ConfiguredComponent.java | 61 +
.../nifi/controller/ContentAvailability.java | 55 +
.../org/apache/nifi/controller/Counter.java | 32 +
.../nifi/controller/EventBasedWorker.java | 32 +
.../org/apache/nifi/controller/Heartbeater.java | 22 +
.../nifi/controller/ProcessScheduler.java | 171 +
.../apache/nifi/controller/ProcessorNode.java | 98 +
.../nifi/controller/ReportingTaskNode.java | 101 +
.../nifi/controller/StandardFlowFileQueue.java | 1104 +++++
.../apache/nifi/controller/StandardFunnel.java | 538 +++
.../controller/ValidationContextFactory.java | 31 +
.../org/apache/nifi/controller/WorkerQueue.java | 36 +
.../exception/CommunicationsException.java | 40 +
.../exception/ComponentLifeCycleException.java | 30 +
...ControllerServiceAlreadyExistsException.java | 26 +
...ControllerServiceInstantiationException.java | 38 +
.../ProcessorInstantiationException.java | 27 +
.../org/apache/nifi/controller/label/Label.java | 48 +
.../ReportingTaskInstantiationException.java | 31 +
.../reporting/ReportingTaskProvider.java | 111 +
.../repository/ContentNotFoundException.java | 47 +
.../repository/CounterRepository.java | 34 +
.../controller/repository/FlowFileEvent.java | 54 +
.../repository/FlowFileEventRepository.java | 45 +
.../repository/RepositoryStatusReport.java | 28 +
.../service/ControllerServiceNode.java | 75 +
.../service/ControllerServiceProvider.java | 176 +
.../service/ControllerServiceReference.java | 46 +
.../service/ControllerServiceState.java | 43 +
.../org/apache/nifi/events/BulletinFactory.java | 72 +
.../nifi/events/BulletinProcessingStrategy.java | 27 +
.../apache/nifi/events/ComponentBulletin.java | 30 +
.../org/apache/nifi/events/SystemBulletin.java | 32 +
.../org/apache/nifi/groups/ProcessGroup.java | 681 +++
.../apache/nifi/groups/ProcessGroupCounts.java | 66 +
.../apache/nifi/groups/RemoteProcessGroup.java | 200 +
.../RemoteProcessGroupPortDescriptor.java | 72 +
.../org/apache/nifi/logging/LogMessage.java | 74 +
.../org/apache/nifi/logging/LogObserver.java | 22 +
.../org/apache/nifi/logging/LogRepository.java | 67 +
.../nifi/logging/LogRepositoryFactory.java | 61 +
.../nifi/remote/PortAuthorizationResult.java | 25 +
.../nifi/remote/RemoteAuthorizationState.java | 27 +
.../org/apache/nifi/remote/RemoteGroupPort.java | 43 +
.../org/apache/nifi/remote/RootGroupPort.java | 78 +
.../nifi/remote/cluster/NodeInformant.java | 22 +
.../remote/exception/BadRequestException.java | 30 +
.../exception/NotAuthorizedException.java | 26 +
.../exception/RequestExpiredException.java | 26 +
.../nifi/remote/protocol/ServerProtocol.java | 141 +
.../nifi-framework-core/.gitignore | 1 +
.../nifi-framework/nifi-framework-core/pom.xml | 144 +
.../apache/nifi/cluster/BulletinsPayload.java | 94 +
.../nifi/cluster/ConnectionException.java | 41 +
.../nifi/cluster/DisconnectionException.java | 41 +
.../apache/nifi/cluster/HeartbeatPayload.java | 168 +
.../org/apache/nifi/connectable/LocalPort.java | 171 +
.../nifi/connectable/StandardConnection.java | 349 ++
.../nifi/controller/EventDrivenWorkerQueue.java | 329 ++
.../nifi/controller/FileSystemSwapManager.java | 769 ++++
.../apache/nifi/controller/FlowController.java | 3766 ++++++++++++++++
.../nifi/controller/FlowFromDOMFactory.java | 458 ++
.../controller/FlowSerializationException.java | 46 +
.../apache/nifi/controller/FlowSerializer.java | 40 +
.../FlowSynchronizationException.java | 45 +
.../nifi/controller/FlowSynchronizer.java | 43 +
.../nifi/controller/FlowUnmarshaller.java | 77 +
.../apache/nifi/controller/SnippetManager.java | 96 +
.../apache/nifi/controller/StandardCounter.java | 108 +
.../nifi/controller/StandardFlowSerializer.java | 453 ++
.../nifi/controller/StandardFlowService.java | 864 ++++
.../controller/StandardFlowSynchronizer.java | 1203 +++++
.../nifi/controller/StandardProcessorNode.java | 1257 ++++++
.../apache/nifi/controller/StandardSnippet.java | 153 +
.../org/apache/nifi/controller/Template.java | 37 +
.../apache/nifi/controller/TemplateManager.java | 523 +++
.../controller/UninheritableFlowException.java | 46 +
.../exception/FlowFileConsumptionException.java | 38 +
.../ReportingTaskInstantiationException.java | 31 +
.../exception/ValidationException.java | 47 +
.../nifi/controller/label/StandardLabel.java | 110 +
.../reporting/AbstractReportingTaskNode.java | 252 ++
.../reporting/StandardReportingContext.java | 143 +
.../StandardReportingInitializationContext.java | 115 +
.../reporting/StandardReportingTaskNode.java | 40 +
.../repository/BatchingSessionFactory.java | 242 +
.../repository/ConnectionSwapInfo.java | 58 +
.../repository/FileSystemRepository.java | 1358 ++++++
.../controller/repository/ProcessContext.java | 279 ++
.../repository/ProvenanceEventEnricher.java | 33 +
.../repository/RepositoryPurgeException.java | 44 +
.../repository/RingBufferEventRepository.java | 312 ++
.../repository/StandardCounterRepository.java | 109 +
.../repository/StandardFlowFileEvent.java | 237 +
.../repository/StandardFlowFileRecord.java | 308 ++
.../repository/StandardProcessSession.java | 2700 +++++++++++
.../StandardProcessSessionFactory.java | 33 +
.../repository/StandardProvenanceReporter.java | 460 ++
.../repository/StandardRepositoryRecord.java | 202 +
.../StandardRepositoryStatusReport.java | 84 +
.../repository/VolatileContentRepository.java | 641 +++
.../repository/VolatileFlowFileRepository.java | 107 +
.../WriteAheadFlowFileRepository.java | 845 ++++
.../repository/claim/ContentDirection.java | 32 +
.../repository/claim/StandardContentClaim.java | 142 +
.../claim/StandardContentClaimManager.java | 145 +
.../repository/io/ArrayManagedOutputStream.java | 245 +
.../repository/io/ByteCountingInputStream.java | 100 +
.../repository/io/ByteCountingOutputStream.java | 62 +
.../io/DisableOnCloseInputStream.java | 91 +
.../io/DisableOnCloseOutputStream.java | 71 +
.../io/FlowFileAccessInputStream.java | 161 +
.../io/FlowFileAccessOutputStream.java | 92 +
.../repository/io/LimitedInputStream.java | 110 +
.../controller/repository/io/LongHolder.java | 46 +
.../controller/repository/io/MemoryManager.java | 55 +
.../repository/io/SyncOnCloseOutputStream.java | 58 +
.../scheduling/ConnectableProcessContext.java | 193 +
.../scheduling/EventDrivenSchedulingAgent.java | 348 ++
.../scheduling/ProcessContextFactory.java | 51 +
.../scheduling/QuartzSchedulingAgent.java | 231 +
.../controller/scheduling/ScheduleState.java | 94 +
.../controller/scheduling/SchedulingAgent.java | 45 +
.../scheduling/StandardProcessScheduler.java | 716 +++
.../scheduling/TimerDrivenSchedulingAgent.java | 237 +
.../service/ControllerServiceLoader.java | 164 +
.../service/StandardConfigurationContext.java | 88 +
...dControllerServiceInitializationContext.java | 82 +
.../service/StandardControllerServiceNode.java | 229 +
.../StandardControllerServiceProvider.java | 628 +++
.../StandardControllerServiceReference.java | 105 +
.../history/StandardMetricDescriptor.java | 109 +
.../status/history/StandardStatusHistory.java | 54 +
.../status/history/StandardStatusSnapshot.java | 78 +
.../status/history/StatusHistoryUtil.java | 107 +
.../VolatileComponentStatusRepository.java | 690 +++
.../tasks/ContinuallyRunConnectableTask.java | 111 +
.../tasks/ContinuallyRunProcessorTask.java | 193 +
.../nifi/controller/tasks/ExpireFlowFiles.java | 112 +
.../controller/tasks/ReportingTaskWrapper.java | 66 +
.../nifi/diagnostics/DiagnosticUtils.java | 27 +
.../nifi/diagnostics/GarbageCollection.java | 60 +
.../apache/nifi/diagnostics/StorageUsage.java | 71 +
.../nifi/diagnostics/SystemDiagnostics.java | 211 +
.../diagnostics/SystemDiagnosticsFactory.java | 132 +
.../nifi/encrypt/EncryptionException.java | 42 +
.../apache/nifi/encrypt/StringEncryptor.java | 148 +
.../java/org/apache/nifi/engine/FlowEngine.java | 127 +
.../events/NodeBulletinProcessingStrategy.java | 66 +
.../nifi/events/VolatileBulletinRepository.java | 305 ++
.../events/network/CommunicationsFailure.java | 36 +
.../nifi/events/network/NetworkTransfer.java | 48 +
.../nifi/fingerprint/FingerprintException.java | 40 +
.../nifi/fingerprint/FingerprintFactory.java | 1055 +++++
.../nifi/groups/StandardProcessGroup.java | 2045 +++++++++
.../org/apache/nifi/jaxb/AdaptedBulletin.java | 110 +
.../org/apache/nifi/jaxb/AdaptedCounter.java | 63 +
.../org/apache/nifi/jaxb/BulletinAdapter.java | 59 +
.../org/apache/nifi/jaxb/CounterAdapter.java | 50 +
.../org/apache/nifi/lifecycle/LifeCycle.java | 46 +
.../nifi/lifecycle/LifeCycleException.java | 46 +
.../nifi/lifecycle/LifeCycleStartException.java | 46 +
.../nifi/lifecycle/LifeCycleStopException.java | 46 +
.../logging/ControllerServiceLogObserver.java | 46 +
.../nifi/logging/ProcessorLogObserver.java | 48 +
.../nifi/logging/ReportingTaskLogObserver.java | 45 +
.../repository/StandardLogRepository.java | 173 +
.../nifi/persistence/FlowConfigurationDAO.java | 99 +
.../StandardSnippetDeserializer.java | 42 +
.../persistence/StandardSnippetSerializer.java | 47 +
.../StandardXMLFlowConfigurationDAO.java | 158 +
.../nifi/persistence/TemplateDeserializer.java | 42 +
.../nifi/persistence/TemplateSerializer.java | 47 +
.../nifi/processor/SimpleProcessLogger.java | 302 ++
.../nifi/processor/StandardProcessContext.java | 182 +
.../StandardProcessorInitializationContext.java | 49 +
.../nifi/processor/StandardPropertyValue.java | 157 +
.../processor/StandardSchedulingContext.java | 114 +
.../processor/StandardValidationContext.java | 136 +
.../StandardValidationContextFactory.java | 44 +
.../org/apache/nifi/remote/RemoteNiFiUtils.java | 203 +
.../nifi/remote/StandardRemoteProcessGroup.java | 1257 ++++++
...tandardRemoteProcessGroupPortDescriptor.java | 148 +
.../org/apache/nifi/services/FlowService.java | 118 +
.../nifi/spring/FlowControllerFactoryBean.java | 101 +
.../spring/RingBufferEventRepositoryBean.java | 45 +
.../spring/StandardFlowServiceFactoryBean.java | 96 +
.../apache/nifi/util/ComponentStatusReport.java | 137 +
.../java/org/apache/nifi/util/Connectables.java | 56 +
.../java/org/apache/nifi/util/DomUtils.java | 89 +
.../org/apache/nifi/util/ReflectionUtils.java | 277 ++
.../java/org/apache/nifi/util/SnippetUtils.java | 239 +
.../src/main/resources/FlowConfiguration.xsd | 374 ++
...nifi.controller.repository.ContentRepository | 16 +
...ifi.controller.repository.FlowFileRepository | 16 +
...fi.controller.repository.FlowFileSwapManager | 15 +
...ler.status.history.ComponentStatusRepository | 15 +
.../src/main/resources/nifi-context.xml | 49 +
.../nifi/cluster/HeartbeatPayloadTest.java | 120 +
.../controller/StandardFlowServiceTest.java | 299 ++
.../controller/TestFileSystemSwapManager.java | 104 +
.../repository/TestFileSystemRepository.java | 352 ++
.../TestRingBufferEventRepository.java | 138 +
.../repository/TestStandardProcessSession.java | 1322 ++++++
.../TestStandardProvenanceReporter.java | 62 +
.../TestVolatileContentRepository.java | 183 +
.../TestWriteAheadFlowFileRepository.java | 135 +
.../repository/io/TestLimitedInputStream.java | 122 +
.../controller/repository/util/DiskUtils.java | 77 +
.../TestStandardProcessScheduler.java | 107 +
.../StandardControllerServiceProviderTest.java | 71 +
.../TestStandardControllerServiceProvider.java | 379 ++
.../controller/service/mock/DummyProcessor.java | 48 +
.../nifi/controller/service/mock/ServiceA.java | 48 +
.../nifi/controller/service/mock/ServiceB.java | 23 +
.../service/util/TestControllerService.java | 61 +
.../fingerprint/FingerprintFactoryTest.java | 59 +
.../processor/TestStandardPropertyValue.java | 176 +
.../processors/DataGeneratorTestProcessor.java | 85 +
.../StubAttributeLoggerProcessor.java | 111 +
.../StubTerminationFileProcessor.java | 40 +
...org.apache.nifi.controller.ControllerService | 15 +
.../src/test/resources/conf/0bytes.xml | 0
.../test/resources/conf/all-flow-corrupt.xml | 201 +
.../resources/conf/all-flow-inheritable.xml | 196 +
.../resources/conf/all-flow-uninheritable.xml | 202 +
.../src/test/resources/conf/all-flow.xml | 198 +
.../src/test/resources/conf/nifi.properties | 127 +
.../test/resources/conf/only-termination.xml | 37 +
.../src/test/resources/conf/remote-flow.xml | 145 +
.../src/test/resources/conf/standard-flow.xml | 196 +
.../src/test/resources/conf/taskConfig.xml | 17 +
.../test/resources/conf/termination-only.xml | 0
.../src/test/resources/hello.txt | 1 +
.../src/test/resources/logback-test.xml | 36 +
.../test/resources/nifi-with-remote.properties | 127 +
.../src/test/resources/nifi.properties | 127 +
.../test/resources/nifi/fingerprint/flow1a.xml | 160 +
.../test/resources/nifi/fingerprint/flow1b.xml | 160 +
.../test/resources/nifi/fingerprint/flow2.xml | 160 +
.../src/test/resources/old-swap-file.swap | Bin 0 -> 1730054 bytes
.../nifi-framework/nifi-nar-utils/.gitignore | 2 +
.../nifi-framework/nifi-nar-utils/pom.xml | 34 +
.../org/apache/nifi/nar/ExtensionManager.java | 187 +
.../org/apache/nifi/nar/ExtensionMapping.java | 73 +
.../org/apache/nifi/nar/NarClassLoader.java | 227 +
.../org/apache/nifi/nar/NarClassLoaders.java | 293 ++
.../java/org/apache/nifi/nar/NarCloseable.java | 44 +
.../nifi/nar/NarThreadContextClassLoader.java | 187 +
.../java/org/apache/nifi/nar/NarUnpacker.java | 396 ++
.../java/org/apache/nifi/util/FileUtils.java | 186 +
.../org/apache/nifi/nar/NarUnpackerTest.java | 214 +
.../resources/NarUnpacker/conf/nifi.properties | 129 +
.../resources/NarUnpacker/lib/dummy-one.nar | Bin 0 -> 1749 bytes
.../NarUnpacker/lib/nifi-framework-nar.nar | Bin 0 -> 406 bytes
.../resources/NarUnpacker/lib2/dummy-two.nar | Bin 0 -> 1751 bytes
.../nifi-framework/nifi-resources/LICENSE | 202 +
.../nifi-framework/nifi-resources/NOTICE | 5 +
.../nifi-framework/nifi-resources/pom.xml | 50 +
.../src/main/assembly/dependencies.xml | 52 +
.../src/main/resources/bin/dump-nifi.bat | 47 +
.../src/main/resources/bin/nifi.sh | 219 +
.../src/main/resources/bin/run-nifi.bat | 47 +
.../src/main/resources/bin/status-nifi.bat | 47 +
.../main/resources/conf/authority-providers.xml | 43 +
.../main/resources/conf/authorized-users.xml | 57 +
.../src/main/resources/conf/bootstrap.conf | 57 +
.../src/main/resources/conf/logback.xml | 156 +
.../src/main/resources/conf/nifi.properties | 162 +
.../nifi-framework/nifi-runtime/.gitignore | 2 +
.../nifi-framework/nifi-runtime/pom.xml | 45 +
.../java/org/apache/nifi/BootstrapListener.java | 377 ++
.../src/main/java/org/apache/nifi/NiFi.java | 228 +
.../main/java/org/apache/nifi/NiFiServer.java | 31 +
.../apache/nifi/util/LimitingInputStream.java | 107 +
.../nifi-framework/nifi-security/.gitignore | 1 +
.../nifi-framework/nifi-security/pom.xml | 39 +
.../util/SslContextCreationException.java | 40 +
.../security/util/SslContextFactory.java | 121 +
.../framework/security/util/SslException.java | 40 +
.../security/util/SslServerSocketFactory.java | 82 +
...SslServerSocketFactoryCreationException.java | 41 +
.../security/util/SslSocketFactory.java | 92 +
.../util/SslSocketFactoryCreationException.java | 40 +
.../security/util/SslContextFactoryTest.java | 68 +
.../src/test/resources/localhost-ks.jks | Bin 0 -> 1298 bytes
.../src/test/resources/localhost-ts.jks | Bin 0 -> 891 bytes
.../src/test/resources/log4j.properties | 21 +
.../nifi-framework/nifi-site-to-site/.gitignore | 2 +
.../nifi-framework/nifi-site-to-site/pom.xml | 75 +
.../util/RemoteProcessGroupUtils.java | 218 +
.../nifi/remote/RemoteResourceFactory.java | 168 +
.../nifi/remote/RemoteResourceManager.java | 227 +
.../apache/nifi/remote/RemoteSiteListener.java | 33 +
.../nifi/remote/SocketRemoteSiteListener.java | 352 ++
.../nifi/remote/StandardRemoteGroupPort.java | 447 ++
.../nifi/remote/StandardRootGroupPort.java | 635 +++
.../exception/UnsupportedCodecException.java | 32 +
.../socket/ClusterManagerServerProtocol.java | 207 +
.../socket/SocketFlowFileServerProtocol.java | 642 +++
.../org.apache.nifi.remote.codec.FlowFileCodec | 15 +
...g.apache.nifi.remote.protocol.ClientProtocol | 15 +
...g.apache.nifi.remote.protocol.ServerProtocol | 16 +
.../remote/TestStandardSiteToSiteProtocol.java | 116 +
.../io/socket/TestSocketChannelStreams.java | 231 +
.../io/socket/ssl/TestSSLSocketChannel.java | 381 ++
.../test/resources/dummy-certs/localhost-ks.jks | Bin 0 -> 1298 bytes
.../test/resources/dummy-certs/localhost-ts.jks | Bin 0 -> 891 bytes
.../src/test/resources/logback-test.xml | 30 +
.../src/test/resources/nifi.properties | 111 +
.../nifi-framework/nifi-user-actions/.gitignore | 1 +
.../nifi-framework/nifi-user-actions/pom.xml | 24 +
.../java/org/apache/nifi/action/Action.java | 121 +
.../java/org/apache/nifi/action/Component.java | 34 +
.../java/org/apache/nifi/action/Operation.java | 37 +
.../component/details/ComponentDetails.java | 26 +
.../component/details/ExtensionDetails.java | 34 +
.../details/RemoteProcessGroupDetails.java | 34 +
.../nifi/action/details/ActionDetails.java | 26 +
.../nifi/action/details/ConfigureDetails.java | 52 +
.../nifi/action/details/ConnectDetails.java | 90 +
.../apache/nifi/action/details/MoveDetails.java | 61 +
.../nifi/action/details/PurgeDetails.java | 45 +
.../nifi-custom-ui-utilities/.gitignore | 1 +
.../nifi-web/nifi-custom-ui-utilities/pom.xml | 34 +
.../HttpServletConfigurationRequestContext.java | 56 +
.../nifi/web/HttpServletRequestContext.java | 100 +
.../web/HttpServletRequestContextConfig.java | 119 +
.../nifi-web/nifi-jetty/.gitignore | 1 +
.../nifi-framework/nifi-web/nifi-jetty/pom.xml | 179 +
.../src/main/assembly/distribution.xml | 42 +
.../org/apache/nifi/web/server/JettyServer.java | 859 ++++
.../server/ServerConfigurationException.java | 36 +
.../org/apache/nifi/web/webdefault.xml | 556 +++
.../nifi-web/nifi-ui-extension/pom.xml | 35 +
.../apache/nifi/ui/extension/UiExtension.java | 48 +
.../nifi/ui/extension/UiExtensionMapping.java | 51 +
.../nifi-web/nifi-web-api/.gitignore | 1 +
.../nifi-web/nifi-web-api/pom.xml | 325 ++
.../org/apache/nifi/aop/MethodProfiler.java | 44 +
.../apache/nifi/audit/ControllerAuditor.java | 255 ++
.../nifi/audit/ControllerServiceAuditor.java | 471 ++
.../org/apache/nifi/audit/FunnelAuditor.java | 138 +
.../java/org/apache/nifi/audit/NiFiAuditor.java | 118 +
.../java/org/apache/nifi/audit/PortAuditor.java | 340 ++
.../apache/nifi/audit/ProcessGroupAuditor.java | 259 ++
.../org/apache/nifi/audit/ProcessorAuditor.java | 398 ++
.../apache/nifi/audit/RelationshipAuditor.java | 406 ++
.../nifi/audit/RemoteProcessGroupAuditor.java | 374 ++
.../apache/nifi/audit/ReportingTaskAuditor.java | 348 ++
.../org/apache/nifi/audit/SnippetAuditor.java | 510 +++
.../nifi/web/ClusterRequestException.java | 38 +
.../IllegalClusterResourceRequestException.java | 32 +
.../org/apache/nifi/web/NiFiCoreException.java | 32 +
.../org/apache/nifi/web/NiFiServiceFacade.java | 1444 ++++++
.../apache/nifi/web/NiFiServiceFacadeLock.java | 177 +
.../nifi/web/StandardNiFiContentAccess.java | 147 +
.../nifi/web/StandardNiFiServiceFacade.java | 3525 +++++++++++++++
.../StandardNiFiWebConfigurationContext.java | 731 +++
.../apache/nifi/web/StandardNiFiWebContext.java | 334 ++
.../nifi/web/api/ApplicationResource.java | 397 ++
.../nifi/web/api/BulletinBoardResource.java | 174 +
.../apache/nifi/web/api/ClusterResource.java | 1147 +++++
.../apache/nifi/web/api/ConnectionResource.java | 905 ++++
.../apache/nifi/web/api/ControllerResource.java | 1253 ++++++
.../nifi/web/api/ControllerServiceResource.java | 1072 +++++
.../org/apache/nifi/web/api/FunnelResource.java | 624 +++
.../apache/nifi/web/api/HistoryResource.java | 528 +++
.../apache/nifi/web/api/InputPortResource.java | 648 +++
.../org/apache/nifi/web/api/LabelResource.java | 664 +++
.../org/apache/nifi/web/api/NodeResource.java | 431 ++
.../apache/nifi/web/api/OutputPortResource.java | 649 +++
.../nifi/web/api/ProcessGroupResource.java | 1383 ++++++
.../apache/nifi/web/api/ProcessorResource.java | 920 ++++
.../apache/nifi/web/api/ProvenanceResource.java | 1316 ++++++
.../web/api/RemoteProcessGroupResource.java | 1076 +++++
.../nifi/web/api/ReportingTaskResource.java | 856 ++++
.../apache/nifi/web/api/SnippetResource.java | 713 +++
.../nifi/web/api/SystemDiagnosticsResource.java | 111 +
.../apache/nifi/web/api/TemplateResource.java | 515 +++
.../apache/nifi/web/api/UserGroupResource.java | 465 ++
.../org/apache/nifi/web/api/UserResource.java | 591 +++
.../api/config/AccessDeniedExceptionMapper.java | 54 +
.../config/AccountNotFoundExceptionMapper.java | 47 +
.../config/AdministrationExceptionMapper.java | 43 +
...ationCredentialsNotFoundExceptionMapper.java | 47 +
.../web/api/config/ClusterExceptionMapper.java | 42 +
...ectingNodeMutableRequestExceptionMapper.java | 49 +
...nectedNodeMutableRequestExceptionMapper.java | 49 +
.../config/IllegalArgumentExceptionMapper.java | 46 +
...alClusterResourceRequestExceptionMapper.java | 47 +
.../IllegalClusterStateExceptionMapper.java | 47 +
.../IllegalNodeDeletionExceptionMapper.java | 46 +
...IllegalNodeDisconnectionExceptionMapper.java | 46 +
.../IllegalNodeReconnectionExceptionMapper.java | 46 +
.../api/config/IllegalStateExceptionMapper.java | 46 +
.../IneligiblePrimaryNodeExceptionMapper.java | 47 +
.../config/InvalidRevisionExceptionMapper.java | 47 +
.../api/config/JsonMappingExceptionMapper.java | 46 +
.../api/config/JsonParseExceptionMapper.java | 46 +
.../config/MutableRequestExceptionMapper.java | 48 +
.../web/api/config/NiFiCoreExceptionMapper.java | 47 +
.../config/NoConnectedNodesExceptionMapper.java | 47 +
.../NoResponseFromNodesExceptionMapper.java | 48 +
.../NodeDisconnectionExceptionMapper.java | 47 +
.../config/NodeReconnectionExceptionMapper.java | 47 +
.../web/api/config/NotFoundExceptionMapper.java | 48 +
.../PrimaryRoleAssignmentExceptionMapper.java | 47 +
.../config/ResourceNotFoundExceptionMapper.java | 48 +
.../SafeModeMutableRequestExceptionMapper.java | 48 +
.../nifi/web/api/config/ThrowableMapper.java | 41 +
.../api/config/UnknownNodeExceptionMapper.java | 47 +
.../api/config/ValidationExceptionMapper.java | 49 +
.../config/WebApplicationExceptionMapper.java | 63 +
.../org/apache/nifi/web/api/dto/DtoFactory.java | 2430 ++++++++++
.../request/BulletinBoardPatternParameter.java | 46 +
.../nifi/web/api/request/ClientIdParameter.java | 44 +
.../api/request/ConnectableTypeParameter.java | 41 +
.../nifi/web/api/request/DateTimeParameter.java | 62 +
.../nifi/web/api/request/DoubleParameter.java | 39 +
.../nifi/web/api/request/IntegerParameter.java | 39 +
.../nifi/web/api/request/LongParameter.java | 39 +
.../ApplicationStartupContextListener.java | 150 +
.../nifi/web/controller/ControllerFacade.java | 1425 ++++++
.../web/controller/StandardSearchContext.java | 64 +
.../org/apache/nifi/web/dao/ConnectionDAO.java | 109 +
.../nifi/web/dao/ControllerServiceDAO.java | 105 +
.../java/org/apache/nifi/web/dao/FunnelDAO.java | 83 +
.../java/org/apache/nifi/web/dao/LabelDAO.java | 74 +
.../java/org/apache/nifi/web/dao/PortDAO.java | 91 +
.../apache/nifi/web/dao/ProcessGroupDAO.java | 86 +
.../org/apache/nifi/web/dao/ProcessorDAO.java | 91 +
.../nifi/web/dao/RemoteProcessGroupDAO.java | 132 +
.../apache/nifi/web/dao/ReportingTaskDAO.java | 85 +
.../org/apache/nifi/web/dao/SnippetDAO.java | 88 +
.../org/apache/nifi/web/dao/TemplateDAO.java | 74 +
.../apache/nifi/web/dao/impl/ComponentDAO.java | 69 +
.../web/dao/impl/StandardConnectionDAO.java | 471 ++
.../dao/impl/StandardControllerServiceDAO.java | 262 ++
.../nifi/web/dao/impl/StandardFunnelDAO.java | 132 +
.../nifi/web/dao/impl/StandardInputPortDAO.java | 268 ++
.../nifi/web/dao/impl/StandardLabelDAO.java | 136 +
.../web/dao/impl/StandardOutputPortDAO.java | 268 ++
.../web/dao/impl/StandardProcessGroupDAO.java | 137 +
.../nifi/web/dao/impl/StandardProcessorDAO.java | 459 ++
.../dao/impl/StandardRemoteProcessGroupDAO.java | 376 ++
.../web/dao/impl/StandardReportingTaskDAO.java | 306 ++
.../nifi/web/dao/impl/StandardSnippetDAO.java | 331 ++
.../nifi/web/dao/impl/StandardTemplateDAO.java | 141 +
.../nifi/web/filter/NodeRequestFilter.java | 124 +
.../apache/nifi/web/filter/RequestLogger.java | 77 +
.../nifi/web/filter/ThreadLocalFilter.java | 53 +
.../org/apache/nifi/web/filter/TimerFilter.java | 69 +
.../ControllerServiceProviderFactoryBean.java | 68 +
.../OptimisticLockingManagerFactoryBean.java | 67 +
.../ReportingTaskProviderFactoryBean.java | 68 +
.../org/apache/nifi/web/util/Availability.java | 32 +
.../nifi/web/util/ClientResponseUtils.java | 48 +
.../org/apache/nifi/web/util/SnippetUtils.java | 609 +++
.../src/main/resources/images/bgNifiLogo.png | Bin 0 -> 4232 bytes
.../src/main/resources/images/nifi16.ico | Bin 0 -> 1150 bytes
.../src/main/resources/nifi-web-api-context.xml | 355 ++
.../src/main/resources/templates/endpoint.hbs | 61 +
.../src/main/resources/templates/example.hbs | 18 +
.../src/main/resources/templates/index.html.hbs | 597 +++
.../src/main/resources/templates/operation.hbs | 104 +
.../src/main/resources/templates/type.hbs | 55 +
.../src/main/webapp/WEB-INF/web.xml | 100 +
.../apache/nifi/integration/NiFiWebApiTest.java | 247 +
.../accesscontrol/AdminAccessControlTest.java | 1010 +++++
.../accesscontrol/DfmAccessControlTest.java | 1324 ++++++
.../ReadOnlyAccessControlTest.java | 982 ++++
.../util/NiFiTestAuthorizationProvider.java | 179 +
.../nifi/integration/util/NiFiTestServer.java | 185 +
.../nifi/integration/util/NiFiTestUser.java | 261 ++
.../integration/util/SourceTestProcessor.java | 49 +
.../util/TerminationTestProcessor.java | 44 +
.../nifi/integration/util/TestPrioritizer.java | 31 +
....apache.nifi.authorization.AuthorityProvider | 15 +
...org.apache.nifi.flowfile.FlowFilePrioritizer | 15 +
.../org.apache.nifi.processor.Processor | 16 +
.../access-control/authority-providers.xml | 24 +
.../access-control/controller-services.xml | 18 +
.../resources/access-control/localhost-ks.jks | Bin 0 -> 2246 bytes
.../resources/access-control/localhost-ts.jks | Bin 0 -> 957 bytes
.../resources/access-control/nifi.properties | 136 +
.../access-control/reporting-tasks.xml | 17 +
.../src/test/resources/logback-test.xml | 36 +
.../nifi-web/nifi-web-content-access/pom.xml | 25 +
.../java/org/apache/nifi/web/ContentAccess.java | 32 +
.../apache/nifi/web/ContentRequestContext.java | 51 +
.../apache/nifi/web/DownloadableContent.java | 62 +
.../nifi-web/nifi-web-content-viewer/.gitignore | 1 +
.../nifi-web/nifi-web-content-viewer/pom.xml | 105 +
.../nifi/web/ContentViewerController.java | 300 ++
.../src/main/resources/META-INF/LICENSE | 232 +
.../src/main/resources/META-INF/NOTICE | 17 +
.../src/main/webapp/WEB-INF/jsp/footer.jsp | 20 +
.../src/main/webapp/WEB-INF/jsp/header.jsp | 96 +
.../src/main/webapp/WEB-INF/jsp/hexview.jsp | 32 +
.../src/main/webapp/WEB-INF/jsp/no-viewer.jsp | 20 +
.../src/main/webapp/WEB-INF/web.xml | 26 +
.../src/main/webapp/css/main.css | 113 +
.../main/webapp/js/hexview/hexview.default.css | 42 +
.../src/main/webapp/js/hexview/hexview.js | 232 +
.../nifi-web/nifi-web-docs/.gitignore | 1 +
.../nifi-web/nifi-web-docs/pom.xml | 62 +
.../nifi/web/docs/DocumentationController.java | 94 +
.../src/main/resources/META-INF/NOTICE | 19 +
.../main/webapp/WEB-INF/jsp/documentation.jsp | 130 +
.../WEB-INF/jsp/no-documentation-found.jsp | 31 +
.../src/main/webapp/WEB-INF/web.xml | 33 +
.../src/main/webapp/css/component-usage.css | 179 +
.../nifi-web-docs/src/main/webapp/css/main.css | 214 +
.../src/main/webapp/images/bgBannerFoot.png | Bin 0 -> 189 bytes
.../src/main/webapp/images/bgHeader.png | Bin 0 -> 1697 bytes
.../src/main/webapp/images/bgTableHeader.png | Bin 0 -> 232 bytes
.../src/main/webapp/js/application.js | 285 ++
.../nifi-web/nifi-web-error/.gitignore | 1 +
.../nifi-web/nifi-web-error/pom.xml | 47 +
.../apache/nifi/web/filter/CatchAllFilter.java | 45 +
.../src/main/webapp/WEB-INF/web.xml | 26 +
.../nifi-web-error/src/main/webapp/index.jsp | 37 +
.../nifi-web-optimistic-locking/.gitignore | 1 +
.../nifi-web-optimistic-locking/pom.xml | 43 +
.../apache/nifi/web/ConfigurationRequest.java | 35 +
.../apache/nifi/web/ConfigurationResult.java | 41 +
.../apache/nifi/web/ConfigurationSnapshot.java | 94 +
.../org/apache/nifi/web/FlowModification.java | 57 +
.../nifi/web/OptimisticLockingManager.java | 53 +
.../web/StandardOptimisticLockingManager.java | 158 +
.../org/apache/nifi/web/UpdateRevision.java | 31 +
.../nifi-web/nifi-web-security/.gitignore | 1 +
.../nifi-web/nifi-web-security/pom.xml | 80 +
.../org/apache/nifi/web/security/DnUtils.java | 85 +
.../web/security/UntrustedProxyException.java | 34 +
.../anonymous/NiFiAnonymousUserFilter.java | 101 +
.../NiFiAuthenticationEntryPoint.java | 69 +
.../authorization/NiFiAuthorizationService.java | 175 +
.../authorization/NodeAuthorizedUserFilter.java | 128 +
.../nifi/web/security/user/NiFiUserDetails.java | 100 +
.../nifi/web/security/user/NiFiUserUtils.java | 91 +
.../x509/SubjectDnX509PrincipalExtractor.java | 32 +
.../security/x509/X509AuthenticationFilter.java | 317 ++
.../security/x509/X509CertificateExtractor.java | 52 +
.../x509/ocsp/CertificateStatusException.java | 42 +
.../x509/ocsp/OcspCertificateValidator.java | 441 ++
.../web/security/x509/ocsp/OcspRequest.java | 69 +
.../nifi/web/security/x509/ocsp/OcspStatus.java | 82 +
.../resources/nifi-web-security-context.xml | 83 +
.../NiFiAuthorizationServiceTest.java | 251 ++
.../nifi-web/nifi-web-ui/.gitignore | 1 +
.../nifi-framework/nifi-web/nifi-web-ui/pom.xml | 656 +++
.../apache/nifi/web/filter/IeEdgeHeader.java | 56 +
.../apache/nifi/web/servlet/DownloadSvg.java | 89 +
.../src/main/resources/META-INF/LICENSE | 434 ++
.../filters/bulletin-board-min.properties | 18 +
.../resources/filters/bulletin-board.properties | 25 +
.../resources/filters/canvas-min.properties | 18 +
.../main/resources/filters/canvas.properties | 67 +
.../resources/filters/cluster-min.properties | 18 +
.../main/resources/filters/cluster.properties | 26 +
.../resources/filters/counters-min.properties | 18 +
.../main/resources/filters/counters.properties | 26 +
.../resources/filters/history-min.properties | 18 +
.../main/resources/filters/history.properties | 27 +
.../resources/filters/provenance-min.properties | 18 +
.../resources/filters/provenance.properties | 26 +
.../resources/filters/summary-min.properties | 18 +
.../main/resources/filters/summary.properties | 32 +
.../resources/filters/templates-min.properties | 18 +
.../main/resources/filters/templates.properties | 26 +
.../main/resources/filters/users-min.properties | 18 +
.../src/main/resources/filters/users.properties | 26 +
.../webapp/WEB-INF/pages/bulletin-board.jsp | 50 +
.../src/main/webapp/WEB-INF/pages/canvas.jsp | 127 +
.../src/main/webapp/WEB-INF/pages/cluster.jsp | 62 +
.../src/main/webapp/WEB-INF/pages/counters.jsp | 61 +
.../src/main/webapp/WEB-INF/pages/history.jsp | 65 +
.../main/webapp/WEB-INF/pages/message-page.jsp | 34 +
.../main/webapp/WEB-INF/pages/provenance.jsp | 68 +
.../src/main/webapp/WEB-INF/pages/summary.jsp | 83 +
.../src/main/webapp/WEB-INF/pages/templates.jsp | 63 +
.../src/main/webapp/WEB-INF/pages/users.jsp | 71 +
.../webapp/WEB-INF/partials/banners-main.jsp | 19 +
.../webapp/WEB-INF/partials/banners-utility.jsp | 19 +
.../bulletin-board/bulletin-board-content.jsp | 40 +
.../WEB-INF/partials/canvas/about-dialog.jsp | 29 +
.../WEB-INF/partials/canvas/canvas-header.jsp | 56 +
.../canvas/connection-configuration.jsp | 180 +
.../partials/canvas/connections-dialog.jsp | 35 +
.../canvas/controller-service-configuration.jsp | 86 +
.../disable-controller-service-dialog.jsp | 76 +
.../canvas/enable-controller-service-dialog.jsp | 75 +
.../partials/canvas/fill-color-dialog.jsp | 46 +
.../WEB-INF/partials/canvas/flow-status.jsp | 55 +
.../canvas/instantiate-template-dialog.jsp | 22 +
.../partials/canvas/label-configuration.jsp | 38 +
.../WEB-INF/partials/canvas/navigation.jsp | 57 +
.../canvas/new-controller-service-dialog.jsp | 53 +
.../WEB-INF/partials/canvas/new-port-dialog.jsp | 27 +
.../canvas/new-process-group-dialog.jsp | 27 +
.../partials/canvas/new-processor-dialog.jsp | 48 +
.../canvas/new-remote-process-group-dialog.jsp | 27 +
.../canvas/new-reporting-task-dialog.jsp | 53 +
.../partials/canvas/new-template-dialog.jsp | 33 +
.../partials/canvas/port-configuration.jsp | 54 +
.../WEB-INF/partials/canvas/port-details.jsp | 39 +
.../canvas/process-group-configuration.jsp | 39 +
.../partials/canvas/process-group-details.jsp | 39 +
.../partials/canvas/processor-configuration.jsp | 206 +
.../WEB-INF/partials/canvas/registration.jsp | 44 +
.../canvas/remote-port-configuration.jsp | 42 +
.../remote-process-group-configuration.jsp | 54 +
.../canvas/remote-process-group-details.jsp | 54 +
.../canvas/remote-process-group-ports.jsp | 47 +
.../canvas/reporting-task-configuration.jsp | 107 +
.../canvas/secure-port-configuration.jsp | 82 +
.../partials/canvas/secure-port-details.jsp | 67 +
.../partials/canvas/settings-content.jsp | 94 +
.../webapp/WEB-INF/partials/canvas/shell.jsp | 27 +
.../partials/cluster/cluster-content.jsp | 39 +
.../partials/cluster/node-details-dialog.jsp | 42 +
.../WEB-INF/partials/connection-details.jsp | 123 +
.../partials/counters/counters-content.jsp | 39 +
.../partials/history/action-details-dialog.jsp | 27 +
.../partials/history/history-content.jsp | 36 +
.../partials/history/history-filter-dialog.jsp | 73 +
.../partials/history/history-purge-dialog.jsp | 42 +
.../webapp/WEB-INF/partials/message-pane.jsp | 23 +
.../main/webapp/WEB-INF/partials/ok-dialog.jsp | 22 +
.../WEB-INF/partials/processor-details.jsp | 154 +
.../provenance/lineage-query-dialog.jsp | 26 +
.../partials/provenance/provenance-content.jsp | 73 +
.../provenance-event-details-dialog.jsp | 196 +
.../provenance/provenance-query-dialog.jsp | 26 +
.../provenance/provenance-search-dialog.jsp | 101 +
.../WEB-INF/partials/status-history-dialog.jsp | 37 +
.../cluster-connection-summary-dialog.jsp | 36 +
.../cluster-input-port-summary-dialog.jsp | 36 +
.../cluster-output-port-summary-dialog.jsp | 36 +
.../cluster-process-group-summary-dialog.jsp | 36 +
.../cluster-processor-summary-dialog.jsp | 36 +
...ster-remote-process-group-summary-dialog.jsp | 36 +
.../partials/summary/summary-content.jsp | 67 +
.../summary/system-diagnostics-dialog.jsp | 131 +
.../summary/view-single-node-dialog.jsp | 24 +
.../partials/templates/templates-content.jsp | 59 +
.../partials/users/group-revoke-dialog.jsp | 22 +
.../partials/users/group-roles-dialog.jsp | 52 +
.../partials/users/user-delete-dialog.jsp | 23 +
.../partials/users/user-details-dialog.jsp | 56 +
.../partials/users/user-group-dialog.jsp | 27 +
.../partials/users/user-revoke-dialog.jsp | 23 +
.../partials/users/user-roles-dialog.jsp | 60 +
.../WEB-INF/partials/users/users-content.jsp | 46 +
.../webapp/WEB-INF/partials/yes-no-dialog.jsp | 22 +
.../nifi-web-ui/src/main/webapp/WEB-INF/web.xml | 153 +
.../nifi-web-ui/src/main/webapp/css/about.css | 53 +
.../nifi-web-ui/src/main/webapp/css/banner.css | 80 +
.../src/main/webapp/css/bulletin-board.css | 238 +
.../nifi-web-ui/src/main/webapp/css/canvas.css | 45 +
.../nifi-web-ui/src/main/webapp/css/cluster.css | 152 +
.../webapp/css/connection-configuration.css | 167 +
.../src/main/webapp/css/connection-details.css | 76 +
.../src/main/webapp/css/controller-service.css | 304 ++
.../src/main/webapp/css/counters.css | 134 +
.../nifi-web-ui/src/main/webapp/css/dialog.css | 332 ++
.../src/main/webapp/css/flow-status.css | 110 +
.../nifi-web-ui/src/main/webapp/css/graph.css | 262 ++
.../nifi-web-ui/src/main/webapp/css/header.css | 683 +++
.../nifi-web-ui/src/main/webapp/css/history.css | 216 +
.../src/main/webapp/css/label-configuration.css | 54 +
.../nifi-web-ui/src/main/webapp/css/main.css | 314 ++
.../src/main/webapp/css/message-page.css | 30 +
.../src/main/webapp/css/message-pane.css | 44 +
.../src/main/webapp/css/navigation.css | 292 ++
.../css/new-controller-service-dialog.css | 152 +
.../main/webapp/css/new-processor-dialog.css | 131 +
.../webapp/css/new-reporting-task-dialog.css | 152 +
.../src/main/webapp/css/port-configuration.css | 190 +
.../src/main/webapp/css/port-details.css | 53 +
.../webapp/css/process-group-configuration.css | 30 +
.../main/webapp/css/process-group-details.css | 22 +
.../main/webapp/css/processor-configuration.css | 235 +
.../src/main/webapp/css/processor-details.css | 92 +
.../src/main/webapp/css/provenance.css | 660 +++
.../src/main/webapp/css/registration.css | 45 +
.../css/remote-process-group-configuration.css | 230 +
.../src/main/webapp/css/reporting-task.css | 116 +
.../nifi-web-ui/src/main/webapp/css/reset.css | 43 +
.../src/main/webapp/css/settings.css | 175 +
.../nifi-web-ui/src/main/webapp/css/shell.css | 98 +
.../src/main/webapp/css/status-history.css | 197 +
.../nifi-web-ui/src/main/webapp/css/summary.css | 851 ++++
.../src/main/webapp/css/templates.css | 229 +
.../nifi-web-ui/src/main/webapp/css/users.css | 254 ++
.../src/main/webapp/images/addConnect.png | Bin 0 -> 1996 bytes
.../src/main/webapp/images/bg-error.png | Bin 0 -> 63766 bytes
.../src/main/webapp/images/bgBannerFoot.png | Bin 0 -> 189 bytes
.../src/main/webapp/images/bgBannerHead.png | Bin 0 -> 422 bytes
.../src/main/webapp/images/bgBreadcrumbs.png | Bin 0 -> 2130 bytes
.../src/main/webapp/images/bgButton.png | Bin 0 -> 234 bytes
.../src/main/webapp/images/bgButtonOver.png | Bin 0 -> 234 bytes
.../src/main/webapp/images/bgButtonSelected.png | Bin 0 -> 166 bytes
.../main/webapp/images/bgButtonSelectedOver.png | Bin 0 -> 168 bytes
.../src/main/webapp/images/bgContextMenu.png | Bin 0 -> 746 bytes
.../src/main/webapp/images/bgControlsInset.png | Bin 0 -> 618 bytes
.../src/main/webapp/images/bgHeader.png | Bin 0 -> 1697 bytes
.../src/main/webapp/images/bgInputText.png | Bin 0 -> 139 bytes
.../src/main/webapp/images/bgNifiLogo.png | Bin 0 -> 4232 bytes
.../src/main/webapp/images/bgNifiToolbar.png | Bin 0 -> 2123 bytes
.../main/webapp/images/bgNifiToolbarEdge.png | Bin 0 -> 541 bytes
.../src/main/webapp/images/bgOutline.png | Bin 0 -> 585 bytes
.../src/main/webapp/images/bgPanCenter.png | Bin 0 -> 362 bytes
.../src/main/webapp/images/bgPanZoom.png | Bin 0 -> 832 bytes
.../webapp/images/bgProcessGroupDetailsArea.png | Bin 0 -> 6596 bytes
.../main/webapp/images/bgProcessorStatArea.png | Bin 0 -> 4062 bytes
.../images/bgRemoteProcessGroupDetailsArea.png | Bin 0 -> 4433 bytes
.../src/main/webapp/images/bgSearchInput.png | Bin 0 -> 221 bytes
.../src/main/webapp/images/bgShellClose.png | Bin 0 -> 169 bytes
.../src/main/webapp/images/bgStatusPanel.png | Bin 0 -> 229 bytes
.../src/main/webapp/images/bgTabContainer.png | Bin 0 -> 234 bytes
.../src/main/webapp/images/bgTableHeader.png | Bin 0 -> 232 bytes
.../main/webapp/images/bgToolbarBtnBorder.png | Bin 0 -> 227 bytes
.../src/main/webapp/images/blueBtnBg-over.jpg | Bin 0 -> 356 bytes
.../src/main/webapp/images/blueBtnBg.jpg | Bin 0 -> 356 bytes
.../src/main/webapp/images/buttonColor.png | Bin 0 -> 1679 bytes
.../src/main/webapp/images/buttonCopy.png | Bin 0 -> 1166 bytes
.../src/main/webapp/images/buttonDelete.png | Bin 0 -> 1837 bytes
.../src/main/webapp/images/buttonDisable.png | Bin 0 -> 1737 bytes
.../src/main/webapp/images/buttonEnable.png | Bin 0 -> 1375 bytes
.../src/main/webapp/images/buttonGroup.png | Bin 0 -> 880 bytes
.../src/main/webapp/images/buttonOutline.png | Bin 0 -> 495 bytes
.../src/main/webapp/images/buttonPanDown.png | Bin 0 -> 554 bytes
.../src/main/webapp/images/buttonPanLeft.png | Bin 0 -> 588 bytes
.../src/main/webapp/images/buttonPanRight.png | Bin 0 -> 530 bytes
.../src/main/webapp/images/buttonPanUp.png | Bin 0 -> 551 bytes
.../src/main/webapp/images/buttonPaste.png | Bin 0 -> 1372 bytes
.../src/main/webapp/images/buttonRefresh.png | Bin 0 -> 915 bytes
.../src/main/webapp/images/buttonRun.png | Bin 0 -> 1163 bytes
.../src/main/webapp/images/buttonStop.png | Bin 0 -> 869 bytes
.../src/main/webapp/images/buttonTemplate.png | Bin 0 -> 1290 bytes
.../src/main/webapp/images/buttonZoom100.png | Bin 0 -> 449 bytes
.../src/main/webapp/images/buttonZoomFit.png | Bin 0 -> 499 bytes
.../src/main/webapp/images/buttonZoomIn.png | Bin 0 -> 435 bytes
.../src/main/webapp/images/buttonZoomOut.png | Bin 0 -> 339 bytes
.../src/main/webapp/images/cluster.png | Bin 0 -> 326 bytes
.../src/main/webapp/images/grayBtnBg-over.jpg | Bin 0 -> 319 bytes
.../src/main/webapp/images/grayBtnBg.jpg | Bin 0 -> 317 bytes
.../nifi-web-ui/src/main/webapp/images/grid.gif | Bin 0 -> 135 bytes
.../src/main/webapp/images/headerTabBg.gif | Bin 0 -> 131 bytes
.../src/main/webapp/images/iconAdminUser.png | Bin 0 -> 1960 bytes
.../src/main/webapp/images/iconAlert.png | Bin 0 -> 1396 bytes
.../src/main/webapp/images/iconAlertDialog.png | Bin 0 -> 1241 bytes
.../src/main/webapp/images/iconAutoRefresh.png | Bin 0 -> 3102 bytes
.../src/main/webapp/images/iconBulletin.png | Bin 0 -> 1066 bytes
.../main/webapp/images/iconBulletinBoard.png | Bin 0 -> 2523 bytes
.../src/main/webapp/images/iconCenterView.png | Bin 0 -> 338 bytes
.../src/main/webapp/images/iconChart.png | Bin 0 -> 510 bytes
.../src/main/webapp/images/iconClose.png | Bin 0 -> 231 bytes
.../src/main/webapp/images/iconCluster.png | Bin 0 -> 2112 bytes
.../src/main/webapp/images/iconClusterSmall.png | Bin 0 -> 757 bytes
.../main/webapp/images/iconCollapseChildren.png | Bin 0 -> 571 bytes
.../main/webapp/images/iconCollapseParents.png | Bin 0 -> 645 bytes
.../src/main/webapp/images/iconColor.png | Bin 0 -> 738 bytes
.../src/main/webapp/images/iconCommit.png | Bin 0 -> 445 bytes
.../src/main/webapp/images/iconConfigure.png | Bin 0 -> 696 bytes
.../src/main/webapp/images/iconConnect.png | Bin 0 -> 589 bytes
.../src/main/webapp/images/iconConnection.png | Bin 0 -> 1517 bytes
.../src/main/webapp/images/iconCopy.png | Bin 0 -> 514 bytes
.../src/main/webapp/images/iconCounter.png | Bin 0 -> 1369 bytes
.../src/main/webapp/images/iconDelete.png | Bin 0 -> 670 bytes
.../src/main/webapp/images/iconDetails.png | Bin 0 -> 549 bytes
.../src/main/webapp/images/iconDisable.png | Bin 0 -> 764 bytes
.../src/main/webapp/images/iconDisconnect.png | Bin 0 -> 569 bytes
.../src/main/webapp/images/iconEdit.png | Bin 0 -> 493 bytes
.../src/main/webapp/images/iconEditButton.png | Bin 0 -> 915 bytes
.../src/main/webapp/images/iconEnable.png | Bin 0 -> 472 bytes
.../main/webapp/images/iconExpandChildren.png | Bin 0 -> 521 bytes
.../main/webapp/images/iconExpandParents.png | Bin 0 -> 585 bytes
.../src/main/webapp/images/iconExport.png | Bin 0 -> 453 bytes
.../main/webapp/images/iconExportLineage.png | Bin 0 -> 405 bytes
.../src/main/webapp/images/iconFlowSettings.png | Bin 0 -> 2624 bytes
.../src/main/webapp/images/iconFunnel.png | Bin 0 -> 1223 bytes
.../src/main/webapp/images/iconGoTo.png | Bin 0 -> 448 bytes
.../src/main/webapp/images/iconHistory.png | Bin 0 -> 2347 bytes
.../src/main/webapp/images/iconInfo.png | Bin 0 -> 550 bytes
.../src/main/webapp/images/iconInputPort.png | Bin 0 -> 1842 bytes
.../main/webapp/images/iconInputPortSmall.png | Bin 0 -> 532 bytes
.../webapp/images/iconIsolatedProcessor.png | Bin 0 -> 1781 bytes
.../src/main/webapp/images/iconLineage.png | Bin 0 -> 432 bytes
.../src/main/webapp/images/iconLoading.gif | Bin 0 -> 2438 bytes
.../src/main/webapp/images/iconMoveToParent.png | Bin 0 -> 215 bytes
.../src/main/webapp/images/iconNotSecure.png | Bin 0 -> 137 bytes
.../src/main/webapp/images/iconOutputPort.png | Bin 0 -> 1658 bytes
.../main/webapp/images/iconOutputPortSmall.png | Bin 0 -> 459 bytes
.../src/main/webapp/images/iconPaste.png | Bin 0 -> 601 bytes
.../src/main/webapp/images/iconPopOut.png | Bin 0 -> 217 bytes
.../webapp/images/iconPortNotTransmitting.png | Bin 0 -> 231 bytes
.../src/main/webapp/images/iconPortRemoved.png | Bin 0 -> 456 bytes
.../main/webapp/images/iconPortTransmitting.png | Bin 0 -> 235 bytes
.../src/main/webapp/images/iconPrimary.png | Bin 0 -> 647 bytes
.../src/main/webapp/images/iconProcessGroup.png | Bin 0 -> 1314 bytes
.../src/main/webapp/images/iconProcessor.png | Bin 0 -> 1446 bytes
.../src/main/webapp/images/iconProvenance.png | Bin 0 -> 1104 bytes
.../src/main/webapp/images/iconRefresh.png | Bin 0 -> 492 bytes
.../src/main/webapp/images/iconRemotePorts.png | Bin 0 -> 456 bytes
.../webapp/images/iconRemoteProcessGroup.png | Bin 0 -> 674 bytes
.../src/main/webapp/images/iconReporting.png | Bin 0 -> 803 bytes
.../src/main/webapp/images/iconResetCounter.png | Bin 0 -> 304 bytes
.../src/main/webapp/images/iconResize.png | Bin 0 -> 142 bytes
.../src/main/webapp/images/iconRevoke.png | Bin 0 -> 676 bytes
.../src/main/webapp/images/iconRun.png | Bin 0 -> 538 bytes
.../src/main/webapp/images/iconSecure.png | Bin 0 -> 133 bytes
.../src/main/webapp/images/iconSmallFunnel.png | Bin 0 -> 455 bytes
.../main/webapp/images/iconSmallInputPort.png | Bin 0 -> 216 bytes
.../main/webapp/images/iconSmallOutputPort.png | Bin 0 -> 264 bytes
.../webapp/images/iconSmallProcessGroup.png | Bin 0 -> 479 bytes
.../main/webapp/images/iconSmallProcessor.png | Bin 0 -> 647 bytes
.../webapp/images/iconSmallRelationship.png | Bin 0 -> 770 bytes
.../images/iconSmallRemoteProcessGroup.png | Bin 0 -> 402 bytes
.../src/main/webapp/images/iconStop.png | Bin 0 -> 402 bytes
.../src/main/webapp/images/iconTemplates.png | Bin 0 -> 2659 bytes
.../src/main/webapp/images/iconToFront.png | Bin 0 -> 475 bytes
.../webapp/images/iconTransmissionActive.png | Bin 0 -> 1330 bytes
.../webapp/images/iconTransmissionInactive.png | Bin 0 -> 1248 bytes
.../src/main/webapp/images/iconTwistArrow.png | Bin 0 -> 256 bytes
.../src/main/webapp/images/iconUsage.png | Bin 0 -> 470 bytes
.../src/main/webapp/images/inputCheckbox.png | Bin 0 -> 389 bytes
.../src/main/webapp/images/loadAnimation.gif | Bin 0 -> 10789 bytes
.../src/main/webapp/images/nifi16.ico | Bin 0 -> 1150 bytes
.../src/main/webapp/images/nifi_about.png | Bin 0 -> 16323 bytes
.../src/main/webapp/images/panelBg.jpg | Bin 0 -> 312 bytes
.../src/main/webapp/images/portRemoved.png | Bin 0 -> 1090 bytes
.../src/main/webapp/images/portRunning.png | Bin 0 -> 337 bytes
.../src/main/webapp/images/portStopped.png | Bin 0 -> 192 bytes
.../src/main/webapp/images/spacer.png | Bin 0 -> 110 bytes
.../src/main/webapp/images/starburst.png | Bin 0 -> 190 bytes
.../src/main/webapp/images/tabBg.jpg | Bin 0 -> 320 bytes
.../src/main/webapp/images/toolbarBg.gif | Bin 0 -> 45 bytes
.../main/webapp/images/toolboxIconFunnel.png | Bin 0 -> 2409 bytes
.../main/webapp/images/toolboxIconInputPort.png | Bin 0 -> 2680 bytes
.../src/main/webapp/images/toolboxIconLabel.png | Bin 0 -> 2871 bytes
.../webapp/images/toolboxIconOutputPort.png | Bin 0 -> 2209 bytes
.../webapp/images/toolboxIconProcessGroup.png | Bin 0 -> 4176 bytes
.../main/webapp/images/toolboxIconProcessor.png | Bin 0 -> 4426 bytes
.../images/toolboxIconRemoteProcessGroup.png | Bin 0 -> 5653 bytes
.../main/webapp/images/toolboxIconTemplate.png | Bin 0 -> 2913 bytes
.../images/transmissionSwitchDisabled.png | Bin 0 -> 1240 bytes
.../webapp/images/transmissionSwitchEnabled.png | Bin 0 -> 1600 bytes
.../src/main/webapp/images/ungroup.png | Bin 0 -> 3409 bytes
.../src/main/webapp/js/codemirror/LICENSE | 19 +
.../js/codemirror/addon/fold/foldgutter.css | 20 +
.../js/codemirror/addon/hint/show-hint.css | 38 +
.../js/codemirror/lib/codemirror-compressed.js | 26 +
.../webapp/js/codemirror/lib/codemirror.css | 318 ++
.../nifi-web-ui/src/main/webapp/js/d3/LICENSE | 26 +
.../nifi-web-ui/src/main/webapp/js/d3/d3.min.js | 5 +
.../webapp/js/jquery/combo/jquery.combo.css | 96 +
.../main/webapp/js/jquery/combo/jquery.combo.js | 311 ++
.../main/webapp/js/jquery/combo/menuArrow.png | Bin 0 -> 251 bytes
.../webapp/js/jquery/combo/menuArrowSmall.png | Bin 0 -> 233 bytes
.../main/webapp/js/jquery/jquery-2.1.1.min.js | 4 +
.../src/main/webapp/js/jquery/jquery.center.js | 35 +
.../src/main/webapp/js/jquery/jquery.count.js | 99 +
.../src/main/webapp/js/jquery/jquery.each.js | 31 +
.../main/webapp/js/jquery/jquery.ellipsis.js | 164 +
.../js/jquery/jquery.event.drag-2.2.min.js | 6 +
.../main/webapp/js/jquery/jquery.form.min.js | 11 +
.../src/main/webapp/js/jquery/jquery.tab.js | 69 +
.../js/jquery/minicolors/jquery.minicolors.css | 259 ++
.../jquery/minicolors/jquery.minicolors.min.js | 9 +
.../js/jquery/minicolors/jquery.minicolors.png | Bin 0 -> 77459 bytes
.../webapp/js/jquery/modal/jquery.modal.css | 112 +
.../main/webapp/js/jquery/modal/jquery.modal.js | 249 ++
.../js/jquery/nfeditor/jquery.nfeditor.css | 73 +
.../js/jquery/nfeditor/jquery.nfeditor.js | 318 ++
.../js/jquery/nfeditor/languages/nfel.css | 51 +
.../webapp/js/jquery/nfeditor/languages/nfel.js | 834 ++++
.../jquery/propertytable/buttonNewProperty.png | Bin 0 -> 590 bytes
.../propertytable/jquery.propertytable.css | 216 +
.../propertytable/jquery.propertytable.js | 1647 +++++++
.../webapp/js/jquery/qtip2/jquery.qtip.min.css | 3 +
.../webapp/js/jquery/qtip2/jquery.qtip.min.js | 5 +
.../webapp/js/jquery/slickgrid/MIT-LICENSE.txt | 20 +
.../js/jquery/slickgrid/css/images/collapse.gif | Bin 0 -> 846 bytes
.../js/jquery/slickgrid/css/images/expand.gif | Bin 0 -> 851 bytes
.../js/jquery/slickgrid/css/images/sort-asc.gif | Bin 0 -> 59 bytes
.../jquery/slickgrid/css/images/sort-desc.gif | Bin 0 -> 59 bytes
.../slickgrid/css/slick-default-theme.css | 135 +
.../js/jquery/slickgrid/css/slick.grid.css | 158 +
.../slickgrid/plugins/slick.autotooltips.js | 83 +
.../plugins/slick.cellrangedecorator.js | 66 +
.../plugins/slick.cellrangeselector.js | 113 +
.../plugins/slick.cellselectionmodel.js | 154 +
.../plugins/slick.rowselectionmodel.js | 187 +
.../webapp/js/jquery/slickgrid/slick.core.js | 467 ++
.../js/jquery/slickgrid/slick.dataview.js | 1126 +++++
.../webapp/js/jquery/slickgrid/slick.editors.js | 512 +++
.../js/jquery/slickgrid/slick.formatters.js | 59 +
.../webapp/js/jquery/slickgrid/slick.grid.js | 3422 ++++++++++++++
.../webapp/js/jquery/tabbs/jquery.tabbs.css | 48 +
.../main/webapp/js/jquery/tabbs/jquery.tabbs.js | 98 +
.../js/jquery/tagcloud/jquery.tagcloud.css | 62 +
.../js/jquery/tagcloud/jquery.tagcloud.js | 226 +
.../ui-smoothness/images/animated-overlay.gif | Bin 0 -> 1738 bytes
.../images/ui-bg_flat_0_aaaaaa_40x100.png | Bin 0 -> 212 bytes
.../images/ui-bg_flat_75_ffffff_40x100.png | Bin 0 -> 208 bytes
.../images/ui-bg_glass_55_fbf9ee_1x400.png | Bin 0 -> 335 bytes
.../images/ui-bg_glass_65_ffffff_1x400.png | Bin 0 -> 207 bytes
.../images/ui-bg_glass_75_dadada_1x400.png | Bin 0 -> 262 bytes
.../images/ui-bg_glass_75_e6e6e6_1x400.png | Bin 0 -> 262 bytes
.../images/ui-bg_glass_95_fef1ec_1x400.png | Bin 0 -> 332 bytes
.../ui-bg_highlight-soft_75_cccccc_1x100.png | Bin 0 -> 280 bytes
.../images/ui-icons_222222_256x240.png | Bin 0 -> 6922 bytes
.../images/ui-icons_2e83ff_256x240.png | Bin 0 -> 4549 bytes
.../images/ui-icons_454545_256x240.png | Bin 0 -> 6992 bytes
.../images/ui-icons_888888_256x240.png | Bin 0 -> 6999 bytes
.../images/ui-icons_cd0a0a_256x240.png | Bin 0 -> 4549 bytes
.../ui-smoothness/jquery-ui-1.10.4.min.css | 7 +
.../ui-smoothness/jquery-ui-1.10.4.min.js | 7 +
.../nifi-web-ui/src/main/webapp/js/json2.js | 475 ++
.../js/nf/bulletin-board/nf-bulletin-board.js | 425 ++
.../src/main/webapp/js/nf/canvas/nf-actions.js | 1184 +++++
.../src/main/webapp/js/nf/canvas/nf-birdseye.js | 343 ++
.../webapp/js/nf/canvas/nf-canvas-header.js | 356 ++
.../webapp/js/nf/canvas/nf-canvas-toolbar.js | 165 +
.../webapp/js/nf/canvas/nf-canvas-toolbox.js | 1133 +++++
.../main/webapp/js/nf/canvas/nf-canvas-utils.js | 1379 ++++++
.../src/main/webapp/js/nf/canvas/nf-canvas.js | 1641 +++++++
.../main/webapp/js/nf/canvas/nf-clipboard.js | 99 +
.../main/webapp/js/nf/canvas/nf-connectable.js | 250 ++
.../js/nf/canvas/nf-connection-configuration.js | 1384 ++++++
.../main/webapp/js/nf/canvas/nf-connection.js | 1536 +++++++
.../main/webapp/js/nf/canvas/nf-context-menu.js | 460 ++
.../js/nf/canvas/nf-controller-service.js | 1906 ++++++++
.../main/webapp/js/nf/canvas/nf-custom-ui.js | 47 +
.../main/webapp/js/nf/canvas/nf-draggable.js | 322 ++
.../src/main/webapp/js/nf/canvas/nf-funnel.js | 282 ++
.../src/main/webapp/js/nf/canvas/nf-go-to.js | 733 +++
.../webapp/js/nf/canvas/nf-graph-control.js | 140 +
.../src/main/webapp/js/nf/canvas/nf-graph.js | 201 +
.../js/nf/canvas/nf-label-configuration.js | 135 +
.../src/main/webapp/js/nf/canvas/nf-label.js | 562 +++
.../js/nf/canvas/nf-port-configuration.js | 173 +
.../main/webapp/js/nf/canvas/nf-port-details.js | 66 +
.../src/main/webapp/js/nf/canvas/nf-port.js | 635 +++
.../nf/canvas/nf-process-group-configuration.js | 110 +
.../js/nf/canvas/nf-process-group-details.js | 66 +
.../webapp/js/nf/canvas/nf-process-group.js | 1067 +++++
.../js/nf/canvas/nf-processor-configuration.js | 782 ++++
.../main/webapp/js/nf/canvas/nf-processor.js | 834 ++++
.../main/webapp/js/nf/canvas/nf-registration.js | 71 +
.../nf-remote-process-group-configuration.js | 130 +
.../canvas/nf-remote-process-group-details.js | 72 +
.../nf/canvas/nf-remote-process-group-ports.js | 528 +++
.../js/nf/canvas/nf-remote-process-group.js | 1063 +++++
.../webapp/js/nf/canvas/nf-reporting-task.js | 735 +++
.../src/main/webapp/js/nf/canvas/nf-search.js | 179 +
.../nf/canvas/nf-secure-port-configuration.js | 384 ++
.../js/nf/canvas/nf-secure-port-details.js | 121 +
.../main/webapp/js/nf/canvas/nf-selectable.js | 61 +
.../src/main/webapp/js/nf/canvas/nf-settings.js | 1752 ++++++++
.../src/main/webapp/js/nf/canvas/nf-snippet.js | 215 +
.../src/main/webapp/js/nf/canvas/nf-storage.js | 139 +
.../webapp/js/nf/canvas/nf-toolbar-action.js | 96 +
.../webapp/js/nf/cluster/nf-cluster-table.js | 620 +++
.../src/main/webapp/js/nf/cluster/nf-cluster.js | 157 +
.../webapp/js/nf/counters/nf-counters-table.js | 306 ++
.../main/webapp/js/nf/counters/nf-counters.js | 156 +
.../webapp/js/nf/history/nf-history-model.js | 221 +
.../webapp/js/nf/history/nf-history-table.js | 446 ++
.../src/main/webapp/js/nf/history/nf-history.js | 157 +
.../src/main/webapp/js/nf/nf-client.js | 59 +
.../src/main/webapp/js/nf/nf-common.js | 997 +++++
.../main/webapp/js/nf/nf-connection-details.js | 470 ++
.../src/main/webapp/js/nf/nf-dialog.js | 130 +
.../src/main/webapp/js/nf/nf-namespace.js | 20 +
.../main/webapp/js/nf/nf-processor-details.js | 270 ++
.../src/main/webapp/js/nf/nf-shell.js | 170 +
.../src/main/webapp/js/nf/nf-status-history.js | 1323 ++++++
.../js/nf/provenance/nf-provenance-lineage.js | 1432 ++++++
.../js/nf/provenance/nf-provenance-table.js | 1367 ++++++
.../webapp/js/nf/provenance/nf-provenance.js | 207 +
.../webapp/js/nf/summary/nf-cluster-search.js | 183 +
.../webapp/js/nf/summary/nf-summary-table.js | 2655 +++++++++++
.../src/main/webapp/js/nf/summary/nf-summary.js | 171 +
.../js/nf/templates/nf-templates-table.js | 335 ++
.../main/webapp/js/nf/templates/nf-templates.js | 238 +
.../main/webapp/js/nf/users/nf-users-table.js | 1075 +++++
.../src/main/webapp/js/nf/users/nf-users.js | 149 +
.../nifi-framework/nifi-web/pom.xml | 71 +
.../nifi-framework/pom.xml | 50 +
nifi-nar-bundles/nifi-framework-bundle/pom.xml | 123 +
.../nifi-geo-bundle/nifi-geo-nar/pom.xml | 33 +
.../src/main/resources/META-INF/NOTICE | 68 +
.../nifi-geo-processors/.gitignore | 1 +
.../nifi-geo-bundle/nifi-geo-processors/pom.xml | 43 +
.../org/apache/nifi/processors/GeoEnrichIP.java | 217 +
.../nifi/processors/maxmind/DatabaseReader.java | 274 ++
.../org.apache.nifi.processor.Processor | 16 +
nifi-nar-bundles/nifi-geo-bundle/pom.xml | 42 +
.../nifi-hadoop-bundle/nifi-hadoop-nar/pom.xml | 35 +
.../src/main/resources/META-INF/NOTICE | 35 +
.../nifi-hdfs-processors/pom.xml | 65 +
.../hadoop/AbstractHadoopProcessor.java | 297 ++
.../hadoop/CreateHadoopSequenceFile.java | 168 +
.../nifi/processors/hadoop/FetchHDFS.java | 125 +
...lowFileStreamUnpackerSequenceFileWriter.java | 170 +
.../apache/nifi/processors/hadoop/GetHDFS.java | 543 +++
.../processors/hadoop/GetHDFSSequenceFile.java | 139 +
.../nifi/processors/hadoop/KeyValueReader.java | 129 +
.../apache/nifi/processors/hadoop/ListHDFS.java | 481 ++
.../apache/nifi/processors/hadoop/PutHDFS.java | 409 ++
.../hadoop/SequenceFileWriterImpl.java | 119 +
.../hadoop/TarUnpackerSequenceFileWriter.java | 54 +
.../nifi/processors/hadoop/ValueReader.java | 115 +
.../hadoop/ZipUnpackerSequenceFileWriter.java | 57 +
.../hadoop/util/ByteFilteringOutputStream.java | 155 +
.../processors/hadoop/util/HDFSListing.java | 83 +
.../hadoop/util/InputStreamWritable.java | 59 +
.../nifi/processors/hadoop/util/LongSerDe.java | 48 +
.../hadoop/util/OutputStreamWritable.java | 83 +
.../hadoop/util/SequenceFileReader.java | 28 +
.../hadoop/util/SequenceFileWriter.java | 38 +
.../processors/hadoop/util/StringSerDe.java | 44 +
.../org.apache.nifi.processor.Processor | 20 +
.../additionalDetails.html | 46 +
.../processors/hadoop/AbstractHadoopTest.java | 90 +
.../nifi/processors/hadoop/GetHDFSTest.java | 155 +
.../nifi/processors/hadoop/PutHDFSTest.java | 173 +
.../hadoop/SimpleHadoopProcessor.java | 30 +
.../hadoop/TestCreateHadoopSequenceFile.java | 176 +
.../nifi/processors/hadoop/TestListHDFS.java | 357 ++
.../src/test/resources/core-site-broken.xml | 25 +
.../src/test/resources/core-site.xml | 25 +
.../resources/testdata/13545312236534130.tar | Bin 0 -> 1505280 bytes
.../resources/testdata/13545423550275052.zip | Bin 0 -> 1500841 bytes
.../resources/testdata/13545479542069498.pkg | Bin 0 -> 1500429 bytes
.../src/test/resources/testdata/randombytes-1 | Bin 0 -> 500000 bytes
.../test/resources/testdata/randombytes-1.gz | Bin 0 -> 500175 bytes
.../src/test/resources/testdata/randombytes-2 | Bin 0 -> 500000 bytes
.../src/test/resources/testdata/randombytes-3 | Bin 0 -> 500000 bytes
nifi-nar-bundles/nifi-hadoop-bundle/pom.xml | 38 +
.../nifi-hadoop-libraries-nar/pom.xml | 55 +
.../src/main/resources/META-INF/LICENSE | 387 ++
.../src/main/resources/META-INF/NOTICE | 300 ++
.../nifi-hadoop-libraries-bundle/pom.xml | 29 +
.../nifi-hl7-bundle/nifi-hl7-nar/pom.xml | 36 +
.../src/main/resources/META-INF/LICENSE | 238 +
.../src/main/resources/META-INF/NOTICE | 29 +
.../nifi-hl7-processors/.gitignore | 1 +
.../nifi-hl7-bundle/nifi-hl7-processors/pom.xml | 120 +
.../processors/hl7/ExtractHL7Attributes.java | 244 +
.../apache/nifi/processors/hl7/RouteHL7.java | 221 +
.../org.apache.nifi.processor.Processor | 16 +
.../hl7/TestExtractHL7Attributes.java | 77 +
.../src/test/resources/hypoglycemia.hl7 | 5 +
nifi-nar-bundles/nifi-hl7-bundle/pom.xml | 33 +
nifi-nar-bundles/nifi-jetty-bundle/pom.xml | 77 +
.../src/main/resources/META-INF/NOTICE | 37 +
.../nifi-kafka-bundle/nifi-kafka-nar/pom.xml | 31 +
.../src/main/resources/META-INF/LICENSE | 299 ++
.../src/main/resources/META-INF/NOTICE | 72 +
.../nifi-kafka-processors/pom.xml | 69 +
.../apache/nifi/processors/kafka/GetKafka.java | 356 ++
.../apache/nifi/processors/kafka/PutKafka.java | 561 +++
.../org.apache.nifi.processor.Processor | 16 +
.../additionalDetails.html | 45 +
.../additionalDetails.html | 45 +
.../nifi/processors/kafka/TestGetKafka.java | 160 +
.../nifi/processors/kafka/TestPutKafka.java | 440 ++
nifi-nar-bundles/nifi-kafka-bundle/pom.xml | 37 +
.../nifi-kite-bundle/nifi-kite-nar/pom.xml | 85 +
.../src/main/resources/META-INF/NOTICE | 51 +
.../nifi-kite-processors/pom.xml | 133 +
.../processors/kite/AbstractKiteProcessor.java | 220 +
.../processors/kite/AvroRecordConverter.java | 320 ++
.../apache/nifi/processors/kite/AvroUtil.java | 38 +
.../nifi/processors/kite/ConvertAvroSchema.java | 339 ++
.../nifi/processors/kite/ConvertCSVToAvro.java | 298 ++
.../nifi/processors/kite/ConvertJSONToAvro.java | 202 +
.../nifi/processors/kite/FailureTracker.java | 83 +
.../processors/kite/StoreInKiteDataset.java | 162 +
.../org.apache.nifi.processor.Processor | 18 +
.../additionalDetails.html | 142 +
.../kite/TestAvroRecordConverter.java | 201 +
.../processors/kite/TestCSVToAvroProcessor.java | 217 +
.../kite/TestConfigurationProperty.java | 76 +
.../processors/kite/TestConvertAvroSchema.java | 216 +
.../nifi/processors/kite/TestGetSchema.java | 100 +
.../kite/TestJSONToAvroProcessor.java | 155 +
.../kite/TestKiteProcessorsCluster.java | 131 +
.../kite/TestKiteStorageProcessor.java | 170 +
.../apache/nifi/processors/kite/TestUtil.java | 103 +
nifi-nar-bundles/nifi-kite-bundle/pom.xml | 44 +
.../nifi-language-translation-nar/pom.xml | 36 +
.../src/main/resources/META-INF/NOTICE | 42 +
.../nifi-yandex-processors/.gitignore | 1 +
.../nifi-yandex-processors/pom.xml | 63 +
.../nifi/processors/yandex/YandexTranslate.java | 332 ++
.../processors/yandex/model/Translation.java | 53 +
.../nifi/processors/yandex/util/Languages.java | 86 +
.../yandex/util/ObjectMapperResolver.java | 48 +
.../org.apache.nifi.processor.Processor | 16 +
.../processors/yandex/TestYandexTranslate.java | 223 +
.../nifi-language-translation-bundle/pom.xml | 48 +
.../nifi-mongodb-nar/pom.xml | 37 +
.../src/main/resources/META-INF/NOTICE | 24 +
.../nifi-mongodb-processors/pom.xml | 66 +
.../mongodb/AbstractMongoProcessor.java | 93 +
.../nifi/processors/mongodb/GetMongo.java | 184 +
.../nifi/processors/mongodb/PutMongo.java | 215 +
.../org.apache.nifi.processor.Processor | 16 +
.../nifi/processors/mongodb/GetMongoTest.java | 203 +
.../nifi/processors/mongodb/PutMongoTest.java | 256 ++
nifi-nar-bundles/nifi-mongodb-bundle/pom.xml | 35 +
.../pom.xml | 54 +
.../nifi/provenance/IndexConfiguration.java | 398 ++
.../PersistentProvenanceRepository.java | 2130 +++++++++
.../provenance/RepositoryConfiguration.java | 299 ++
.../nifi/provenance/StandardRecordReader.java | 507 +++
.../nifi/provenance/StandardRecordWriter.java | 343 ++
.../provenance/expiration/ExpirationAction.java | 35 +
.../expiration/FileRemovalAction.java | 65 +
.../provenance/lucene/DeleteIndexAction.java | 120 +
.../nifi/provenance/lucene/DocsReader.java | 192 +
.../nifi/provenance/lucene/FieldNames.java | 24 +
.../nifi/provenance/lucene/IndexManager.java | 512 +++
.../nifi/provenance/lucene/IndexSearch.java | 116 +
.../nifi/provenance/lucene/IndexingAction.java | 133 +
.../nifi/provenance/lucene/LineageQuery.java | 119 +
.../nifi/provenance/lucene/LuceneUtil.java | 163 +
.../provenance/serialization/RecordReader.java | 103 +
.../provenance/serialization/RecordReaders.java | 123 +
.../provenance/serialization/RecordWriter.java | 95 +
.../provenance/serialization/RecordWriters.java | 39 +
.../nifi/provenance/toc/StandardTocReader.java | 155 +
.../nifi/provenance/toc/StandardTocWriter.java | 120 +
.../apache/nifi/provenance/toc/TocReader.java | 71 +
.../org/apache/nifi/provenance/toc/TocUtil.java | 40 +
.../apache/nifi/provenance/toc/TocWriter.java | 53 +
...he.nifi.provenance.ProvenanceEventRepository | 15 +
.../TestPersistentProvenanceRepository.java | 1323 ++++++
.../TestStandardRecordReaderWriter.java | 189 +
.../org/apache/nifi/provenance/TestUtil.java | 82 +
.../provenance/toc/TestStandardTocReader.java | 118 +
.../provenance/toc/TestStandardTocWriter.java | 42 +
.../nifi-provenance-repository-nar/pom.xml | 34 +
.../src/main/resources/META-INF/NOTICE | 202 +
.../nifi-volatile-provenance-repository/pom.xml | 42 +
.../VolatileProvenanceRepository.java | 745 ++++
...he.nifi.provenance.ProvenanceEventRepository | 15 +
.../TestVolatileProvenanceRepository.java | 178 +
.../src/test/resources/nifi.properties | 18 +
.../nifi-provenance-repository-bundle/pom.xml | 43 +
.../nifi-social-media-nar/pom.xml | 36 +
.../src/main/resources/META-INF/NOTICE | 57 +
.../nifi-twitter-processors/.gitignore | 1 +
.../nifi-twitter-processors/pom.xml | 58 +
.../nifi/processors/twitter/GetTwitter.java | 362 ++
.../org.apache.nifi.processor.Processor | 16 +
.../nifi-social-media-bundle/pom.xml | 33 +
.../nifi-solr-bundle/nifi-solr-nar/pom.xml | 36 +
.../src/main/resources/META-INF/LICENSE | 212 +
.../src/main/resources/META-INF/NOTICE | 51 +
.../nifi-solr-processors/pom.xml | 117 +
.../apache/nifi/processors/solr/GetSolr.java | 323 ++
.../processors/solr/PutSolrContentStream.java | 295 ++
.../nifi/processors/solr/SolrProcessor.java | 145 +
.../org.apache.nifi.processor.Processor | 16 +
.../additionalDetails.html | 48 +
.../solr/EmbeddedSolrServerFactory.java | 85 +
.../nifi/processors/solr/TestGetSolr.java | 221 +
.../solr/TestPutSolrContentStream.java | 416 ++
.../src/test/resources/log4j.properties | 14 +
.../src/test/resources/solr/solr.xml | 18 +
.../testCollection/conf/_rest_managed.json | 3 +
.../testCollection/conf/lang/stopwords_en.txt | 54 +
.../resources/testCollection/conf/protwords.txt | 21 +
.../resources/testCollection/conf/schema.xml | 21 +
.../testCollection/conf/solrconfig.xml | 20 +
.../resources/testCollection/conf/synonyms.txt | 29 +
.../resources/testCollection/core.properties | 1 +
.../testdata/test-csv-multiple-docs.csv | 2 +
.../testdata/test-custom-json-single-doc.json | 15 +
.../testdata/test-solr-json-multiple-docs.json | 18 +
.../testdata/test-xml-multiple-docs.xml | 18 +
nifi-nar-bundles/nifi-solr-bundle/pom.xml | 50 +
.../nifi-standard-content-viewer/pom.xml | 75 +
.../web/StandardContentViewerController.java | 100 +
.../src/main/resources/META-INF/NOTICE | 19 +
.../main/webapp/META-INF/nifi-content-viewer | 17 +
.../src/main/webapp/WEB-INF/jsp/codemirror.jsp | 50 +
.../src/main/webapp/WEB-INF/web.xml | 29 +
.../src/main/webapp/css/main.css | 20 +
.../nifi-standard-nar/pom.xml | 49 +
.../src/main/resources/META-INF/LICENSE | 321 ++
.../src/main/resources/META-INF/NOTICE | 155 +
.../nifi-standard-prioritizers/pom.xml | 36 +
.../prioritizer/FirstInFirstOutPrioritizer.java | 37 +
.../NewestFlowFileFirstPrioritizer.java | 37 +
.../OldestFlowFileFirstPrioritizer.java | 37 +
.../PriorityAttributePrioritizer.java | 87 +
...org.apache.nifi.flowfile.FlowFilePrioritizer | 18 +
.../prioritizer/NewestFirstPrioritizerTest.java | 66 +
.../prioritizer/OldestFirstPrioritizerTest.java | 66 +
.../PriorityAttributePrioritizerTest.java | 118 +
.../nifi-standard-processors/pom.xml | 284 ++
.../standard/AbstractJsonPathProcessor.java | 135 +
.../standard/Base64EncodeContent.java | 151 +
.../nifi/processors/standard/BinFiles.java | 351 ++
.../processors/standard/CompressContent.java | 324 ++
.../nifi/processors/standard/ControlRate.java | 397 ++
.../standard/ConvertCharacterSet.java | 176 +
.../processors/standard/DetectDuplicate.java | 260 ++
.../processors/standard/DistributeLoad.java | 484 ++
.../processors/standard/DuplicateFlowFile.java | 81 +
.../nifi/processors/standard/EncodeContent.java | 259 ++
.../processors/standard/EncryptContent.java | 304 ++
.../processors/standard/EvaluateJsonPath.java | 323 ++
.../standard/EvaluateRegularExpression.java | 286 ++
.../nifi/processors/standard/EvaluateXPath.java | 450 ++
.../processors/standard/EvaluateXQuery.java | 446 ++
.../processors/standard/ExecuteProcess.java | 512 +++
.../nifi/processors/standard/ExecuteSQL.java | 157 +
.../standard/ExecuteStreamCommand.java | 390 ++
.../nifi/processors/standard/ExtractText.java | 344 ++
.../processors/standard/GenerateFlowFile.java | 167 +
.../apache/nifi/processors/standard/GetFTP.java | 93 +
.../nifi/processors/standard/GetFile.java | 475 ++
.../processors/standard/GetFileTransfer.java | 305 ++
.../nifi/processors/standard/GetHTTP.java | 533 +++
.../nifi/processors/standard/GetJMSQueue.java | 77 +
.../nifi/processors/standard/GetJMSTopic.java | 348 ++
.../nifi/processors/standard/GetSFTP.java | 109 +
.../processors/standard/HandleHttpRequest.java | 639 +++
.../processors/standard/HandleHttpResponse.java | 186 +
.../nifi/processors/standard/HashAttribute.java | 252 ++
.../nifi/processors/standard/HashContent.java | 159 +
.../processors/standard/IdentifyMimeType.java | 159 +
.../nifi/processors/standard/InvokeHTTP.java | 749 ++++
.../nifi/processors/standard/JmsConsumer.java | 239 +
.../nifi/processors/standard/ListenHTTP.java | 321 ++
.../nifi/processors/standard/ListenUDP.java | 594 +++
.../nifi/processors/standard/LogAttribute.java | 265 ++
.../nifi/processors/standard/MergeContent.java | 1075 +++++
.../nifi/processors/standard/ModifyBytes.java | 131 +
.../processors/standard/MonitorActivity.java | 224 +
.../nifi/processors/standard/PostHTTP.java | 966 ++++
.../nifi/processors/standard/PutEmail.java | 436 ++
.../apache/nifi/processors/standard/PutFTP.java | 161 +
.../nifi/processors/standard/PutFile.java | 382 ++
.../processors/standard/PutFileTransfer.java | 313 ++
.../apache/nifi/processors/standard/PutJMS.java | 374 ++
.../nifi/processors/standard/PutSFTP.java | 90 +
.../nifi/processors/standard/ReplaceText.java | 307 ++
.../standard/ReplaceTextWithMapping.java | 388 ++
.../processors/standard/RouteOnAttribute.java | 272 ++
.../processors/standard/RouteOnContent.java | 241 +
.../nifi/processors/standard/ScanAttribute.java | 238 +
.../nifi/processors/standard/ScanContent.java | 305 ++
.../processors/standard/SegmentContent.java | 188 +
.../nifi/processors/standard/SplitContent.java | 333 ++
.../nifi/processors/standard/SplitJson.java | 200 +
.../nifi/processors/standard/SplitText.java | 374 ++
.../nifi/processors/standard/SplitXml.java | 310 ++
.../nifi/processors/standard/TransformXml.java | 204 +
.../nifi/processors/standard/UnpackContent.java | 442 ++
.../nifi/processors/standard/ValidateXml.java | 153 +
.../servlets/ContentAcknowledgmentServlet.java | 133 +
.../standard/servlets/ListenHTTPServlet.java | 322 ++
.../nifi/processors/standard/util/Bin.java | 162 +
.../processors/standard/util/BinManager.java | 235 +
.../standard/util/DocumentReaderCallback.java | 65 +
.../processors/standard/util/FTPTransfer.java | 586 +++
.../nifi/processors/standard/util/FTPUtils.java | 300 ++
.../nifi/processors/standard/util/FileInfo.java | 167 +
.../processors/standard/util/FileTransfer.java | 259 ++
.../standard/util/FlowFileSessionWrapper.java | 44 +
.../processors/standard/util/JdbcCommon.java | 165 +
.../processors/standard/util/JmsFactory.java | 434 ++
.../standard/util/JmsProcessingSummary.java | 81 +
.../processors/standard/util/JmsProperties.java | 180 +
.../util/JsonPathExpressionValidator.java | 503 +++
.../standard/util/NLKBufferedReader.java | 181 +
.../standard/util/OpenPGPKeyBasedEncryptor.java | 277 ++
.../util/OpenPGPPasswordBasedEncryptor.java | 175 +
.../standard/util/PasswordBasedEncryptor.java | 155 +
.../standard/util/SFTPConnection.java | 73 +
.../processors/standard/util/SFTPTransfer.java | 623 +++
.../processors/standard/util/SFTPUtils.java | 312 ++
.../standard/util/SocksProxySocketFactory.java | 69 +
.../standard/util/UDPStreamConsumer.java | 216 +
.../util/ValidatingBase32InputStream.java | 76 +
.../util/ValidatingBase64InputStream.java | 75 +
.../standard/util/WrappedMessageConsumer.java | 77 +
.../standard/util/WrappedMessageProducer.java | 77 +
.../standard/util/XmlElementNotifier.java | 22 +
.../standard/util/XmlSplitterSaxParser.java | 112 +
.../org.apache.nifi.processor.Processor | 69 +
.../additionalDetails.html | 30 +
.../additionalDetails.html | 31 +
.../additionalDetails.html | 149 +
.../additionalDetails.html | 43 +
.../additionalDetails.html | 44 +
.../additionalDetails.html | 59 +
.../additionalDetails.html | 46 +
.../org/apache/tika/mime/custom-mimetypes.xml | 97 +
.../src/test/java/TestDynamicEnvironment.java | 29 +
.../src/test/java/TestIngestAndUpdate.java | 34 +
.../src/test/java/TestSuccess.java | 24 +
.../processors/standard/CaptureServlet.java | 55 +
.../processors/standard/HelloWorldServlet.java | 42 +
.../standard/RESTServiceContentModified.java | 78 +
.../standard/TestBase64EncodeContent.java | 82 +
.../standard/TestCompressContent.java | 179 +
.../processors/standard/TestControlRate.java | 67 +
.../standard/TestConvertCharacterSet.java | 44 +
.../standard/TestDetectDuplicate.java | 215 +
.../processors/standard/TestDistributeLoad.java | 138 +
.../standard/TestDuplicateFlowFile.java | 36 +
.../processors/standard/TestEncodeContent.java | 164 +
.../processors/standard/TestEncryptContent.java | 163 +
.../standard/TestEvaluateJsonPath.java | 366 ++
.../processors/standard/TestEvaluateXPath.java | 158 +
.../processors/standard/TestEvaluateXQuery.java | 651 +++
.../processors/standard/TestExecuteProcess.java | 160 +
.../processors/standard/TestExecuteSQL.java | 169 +
.../standard/TestExecuteStreamCommand.java | 232 +
.../processors/standard/TestExtractText.java | 313 ++
.../nifi/processors/standard/TestGetFile.java | 186 +
.../nifi/processors/standard/TestGetHTTP.java | 340 ++
.../processors/standard/TestGetJMSQueue.java | 140 +
.../standard/TestHandleHttpRequest.java | 128 +
.../standard/TestHandleHttpResponse.java | 180 +
.../processors/standard/TestHashAttribute.java | 99 +
.../processors/standard/TestHashContent.java | 66 +
.../standard/TestIdentifyMimeType.java | 106 +
.../processors/standard/TestInvokeHTTP.java | 672 +++
.../processors/standard/TestJmsConsumer.java | 171 +
.../nifi/processors/standard/TestListenUDP.java | 213 +
.../processors/standard/TestMergeContent.java | 846 ++++
.../processors/standard/TestModifyBytes.java | 189 +
.../standard/TestMonitorActivity.java | 191 +
.../nifi/processors/standard/TestPostHTTP.java | 248 ++
.../nifi/processors/standard/TestPutEmail.java | 82 +
.../processors/standard/TestReplaceText.java | 389 ++
.../standard/TestReplaceTextLineByLine.java | 336 ++
.../standard/TestReplaceTextWithMapping.java | 332 ++
.../standard/TestRouteOnAttribute.java | 160 +
.../processors/standard/TestRouteOnContent.java | 72 +
.../processors/standard/TestScanAttribute.java | 148 +
.../processors/standard/TestScanContent.java | 103 +
.../processors/standard/TestSegmentContent.java | 64 +
.../nifi/processors/standard/TestServer.java | 159 +
.../processors/standard/TestSplitContent.java | 353 ++
.../nifi/processors/standard/TestSplitJson.java | 214 +
.../nifi/processors/standard/TestSplitText.java | 163 +
.../nifi/processors/standard/TestSplitXml.java | 102 +
.../processors/standard/TestTransformXml.java | 115 +
.../processors/standard/TestUnpackContent.java | 224 +
.../processors/standard/TestValidateXml.java | 41 +
.../standard/UserAgentTestingServlet.java | 41 +
.../standard/util/TestJdbcCommon.java | 155 +
.../standard/util/TestJdbcHugeStream.java | 267 ++
.../CharacterSetConversionSamples/Converted.txt | 1 +
.../Converted2.txt | Bin 0 -> 134580 bytes
.../CharacterSetConversionSamples/Original.txt | 418 ++
.../resources/CompressedData/SampleFile.txt | 418 ++
.../resources/CompressedData/SampleFile.txt.bz2 | Bin 0 -> 388 bytes
.../resources/CompressedData/SampleFile.txt.gz | Bin 0 -> 321 bytes
.../CompressedData/SampleFile1.txt.bz2 | Bin 0 -> 154 bytes
.../resources/CompressedData/SampleFile1.txt.gz | Bin 0 -> 321 bytes
.../CompressedData/SampleFileConcat.txt | 835 ++++
.../CompressedData/SampleFileConcat.txt.bz2 | Bin 0 -> 542 bytes
.../test/resources/ExecuteCommand/1000bytes.txt | 1 +
.../ExecuteCommand/TestDynamicEnvironment.jar | Bin 0 -> 1200 bytes
.../ExecuteCommand/TestIngestAndUpdate.jar | Bin 0 -> 1170 bytes
.../resources/ExecuteCommand/TestSuccess.jar | Bin 0 -> 827 bytes
.../src/test/resources/ExecuteCommand/test.txt | 2 +
.../dictionary-with-empty-new-lines | 5 +
.../ScanAttribute/dictionary-with-extra-info | 9 +
.../test/resources/ScanAttribute/dictionary1 | 5 +
.../test/resources/TestEncryptContent/text.txt | 17 +
.../resources/TestEncryptContent/text.txt.asc | 33 +
.../test/resources/TestIdentifyMimeType/1.7z | Bin 0 -> 133 bytes
.../test/resources/TestIdentifyMimeType/1.jar | Bin 0 -> 466 bytes
.../test/resources/TestIdentifyMimeType/1.mdb | Bin 0 -> 237568 bytes
.../test/resources/TestIdentifyMimeType/1.pdf | Bin 0 -> 277032 bytes
.../test/resources/TestIdentifyMimeType/1.tar | Bin 0 -> 10240 bytes
.../resources/TestIdentifyMimeType/1.tar.gz | Bin 0 -> 154 bytes
.../test/resources/TestIdentifyMimeType/1.txt | 1 +
.../resources/TestIdentifyMimeType/1.txt.bz2 | Bin 0 -> 59 bytes
.../resources/TestIdentifyMimeType/1.txt.gz | Bin 0 -> 50 bytes
.../test/resources/TestIdentifyMimeType/1.xml | 20 +
.../test/resources/TestIdentifyMimeType/1.zip | Bin 0 -> 165 bytes
.../TestIdentifyMimeType/bgBannerFoot.png | Bin 0 -> 189 bytes
.../TestIdentifyMimeType/blueBtnBg.jpg | Bin 0 -> 356 bytes
.../TestIdentifyMimeType/flowfilev1.tar | Bin 0 -> 10240 bytes
.../resources/TestIdentifyMimeType/flowfilev3 | Bin 0 -> 40 bytes
.../resources/TestIdentifyMimeType/grid.gif | Bin 0 -> 135 bytes
.../resources/TestJson/control-characters.json | 7 +
.../test/resources/TestJson/json-sample.json | 415 ++
.../test/resources/TestMergeContent/demarcate | 1 +
.../src/test/resources/TestMergeContent/foot | 1 +
.../src/test/resources/TestMergeContent/head | 1 +
.../test/resources/TestMergeContent/place.avsc | 7 +
.../test/resources/TestMergeContent/user.avsc | 9 +
.../test/resources/TestModifyBytes/noFooter.txt | 10 +
.../TestModifyBytes/noFooter_noHeader.txt | 10 +
.../test/resources/TestModifyBytes/noHeader.txt | 11 +
.../test/resources/TestModifyBytes/testFile.txt | 11 +
.../TestReplaceTextLineByLine/$1$1.txt | 11 +
.../BRue_cRue_RiRey.txt | 11 +
.../TestReplaceTextLineByLine/Blu$2e_clu$2e.txt | 11 +
.../TestReplaceTextLineByLine/D$d_h$d.txt | 11 +
.../TestReplaceTextLineByLine/Good.txt | 1 +
.../TestReplaceTextLineByLine/Spider.txt | 11 +
.../TestReplaceTextLineByLine/[DODO].txt | 11 +
.../TestReplaceTextLineByLine/cu[$1]_Po[$1].txt | 11 +
.../TestReplaceTextLineByLine/cu_Po.txt | 11 +
.../TestReplaceTextLineByLine/food.txt | 11 +
.../TestReplaceTextLineByLine/testFile.txt | 11 +
.../color-fruit-backreference-mapping.txt | 7 +
.../color-fruit-blank-mapping.txt | 7 +
.../color-fruit-escaped-dollar-mapping.txt | 7 +
...t-excessive-backreference-mapping-simple.txt | 6 +
...or-fruit-excessive-backreference-mapping.txt | 6 +
...olor-fruit-invalid-backreference-mapping.txt | 7 +
.../color-fruit-mapping.txt | 7 +
.../color-fruit-no-match-mapping.txt | 7 +
.../color-fruit-space-mapping.txt | 6 +
.../colors-without-dashes.txt | 4 +
.../TestReplaceTextWithMapping/colors.txt | 4 +
.../test/resources/TestScanContent/helloWorld | 1 +
.../resources/TestScanContent/wellthengood-bye | 1 +
.../src/test/resources/TestSplitText/1.txt | 5 +
.../src/test/resources/TestSplitText/2.txt | 5 +
.../src/test/resources/TestSplitText/3.txt | 5 +
.../src/test/resources/TestSplitText/4.txt | 3 +
.../src/test/resources/TestSplitText/5.txt | 7 +
.../src/test/resources/TestSplitText/6.txt | 7 +
.../test/resources/TestSplitText/original.txt | 12 +
.../test/resources/TestTransformXml/math.html | 8 +
.../test/resources/TestTransformXml/math.xml | 21 +
.../test/resources/TestTransformXml/math.xsl | 36 +
.../test/resources/TestTransformXml/tokens.csv | 2 +
.../test/resources/TestTransformXml/tokens.xml | 17 +
.../test/resources/TestTransformXml/tokens.xsl | 103 +
.../resources/TestUnpackContent/data.flowfilev2 | Bin 0 -> 255 bytes
.../resources/TestUnpackContent/data.flowfilev3 | Bin 0 -> 357 bytes
.../test/resources/TestUnpackContent/data.tar | Bin 0 -> 3584 bytes
.../test/resources/TestUnpackContent/data.zip | Bin 0 -> 359 bytes
.../resources/TestUnpackContent/folder/cal.txt | 8 +
.../resources/TestUnpackContent/folder/date.txt | 1 +
.../src/test/resources/TestXml/XmlBundle.xsd | 34 +
.../src/test/resources/TestXml/fruit.xml | 47 +
.../src/test/resources/TestXml/subNode.xml | 21 +
.../src/test/resources/TestXml/xml-bundle-1 | 51 +
.../src/test/resources/TestXml/xml-snippet.xml | 25 +
.../src/test/resources/hello.txt | 1 +
.../src/test/resources/localhost-ks.jks | Bin 0 -> 1298 bytes
.../src/test/resources/localhost-ts.jks | Bin 0 -> 891 bytes
.../src/test/resources/localhost.cer | Bin 0 -> 829 bytes
.../src/test/resources/logback-test.xml | 49 +
.../nifi-standard-reporting-tasks/pom.xml | 60 +
.../ControllerStatusReportingTask.java | 343 ++
.../nifi/controller/MonitorDiskUsage.java | 118 +
.../apache/nifi/controller/MonitorMemory.java | 257 ++
.../ganglia/StandardGangliaReporter.java | 269 ++
.../org.apache.nifi.reporting.ReportingTask | 18 +
.../additionalDetails.html | 70 +
.../additionalDetails.html | 41 +
.../nifi/controller/TestMonitorDiskUsage.java | 61 +
nifi-nar-bundles/nifi-standard-bundle/pom.xml | 65 +
.../nifi-dbcp-service-api/pom.xml | 31 +
.../java/org/apache/nifi/dbcp/DBCPService.java | 34 +
.../nifi-dbcp-service-nar/pom.xml | 37 +
.../src/main/resources/META-INF/LICENSE | 239 +
.../src/main/resources/META-INF/NOTICE | 35 +
.../nifi-dbcp-service/pom.xml | 61 +
.../apache/nifi/dbcp/DBCPConnectionPool.java | 239 +
.../nifi/dbcp/DatabaseSystemDescriptor.java | 51 +
.../org/apache/nifi/dbcp/DatabaseSystems.java | 83 +
.../java/org/apache/nifi/dbcp/DriverShim.java | 74 +
...org.apache.nifi.controller.ControllerService | 15 +
.../org/apache/nifi/dbcp/DBCPServiceTest.java | 289 ++
.../apache/nifi/dbcp/TestDatabaseSystems.java | 31 +
.../org/apache/nifi/dbcp/TestProcessor.java | 45 +
.../nifi-dbcp-service-bundle/pom.xml | 29 +
.../pom.xml | 31 +
.../distributed/cache/client/Deserializer.java | 41 +
.../cache/client/DistributedMapCacheClient.java | 140 +
.../cache/client/DistributedSetCacheClient.java | 78 +
.../distributed/cache/client/Serializer.java | 41 +
.../exception/DeserializationException.java | 33 +
.../exception/SerializationException.java | 33 +
.../pom.xml | 51 +
.../distributed/cache/client/CommsSession.java | 46 +
.../DistributedMapCacheClientService.java | 331 ++
.../DistributedSetCacheClientService.java | 220 +
.../cache/client/SSLCommsSession.java | 109 +
.../cache/client/StandardCommsSession.java | 125 +
...org.apache.nifi.controller.ControllerService | 16 +
.../nifi-distributed-cache-protocol/pom.xml | 33 +
.../cache/protocol/ProtocolHandshake.java | 116 +
.../protocol/exception/HandshakeException.java | 28 +
.../nifi-distributed-cache-server/pom.xml | 70 +
.../cache/server/AbstractCacheServer.java | 204 +
.../distributed/cache/server/CacheRecord.java | 57 +
.../distributed/cache/server/CacheServer.java | 29 +
.../cache/server/DistributedCacheServer.java | 114 +
.../cache/server/DistributedSetCacheServer.java | 73 +
.../cache/server/EvictionPolicy.java | 77 +
.../cache/server/SetCacheServer.java | 105 +
.../server/map/DistributedMapCacheServer.java | 78 +
.../distributed/cache/server/map/MapCache.java | 35 +
.../cache/server/map/MapCacheRecord.java | 59 +
.../cache/server/map/MapCacheServer.java | 153 +
.../cache/server/map/MapPutResult.java | 60 +
.../cache/server/map/PersistentMapCache.java | 232 +
.../cache/server/map/SimpleMapCache.java | 188 +
.../cache/server/set/PersistentSetCache.java | 195 +
.../distributed/cache/server/set/SetCache.java | 32 +
.../cache/server/set/SetCacheRecord.java | 56 +
.../cache/server/set/SetCacheResult.java | 42 +
.../cache/server/set/SimpleSetCache.java | 118 +
...org.apache.nifi.controller.ControllerService | 16 +
.../cache/server/TestServerAndClient.java | 536 +++
.../src/test/resources/localhost-ks.jks | Bin 0 -> 1298 bytes
.../src/test/resources/localhost-ts.jks | Bin 0 -> 891 bytes
.../nifi-distributed-cache-services-nar/pom.xml | 43 +
.../src/main/resources/META-INF/NOTICE | 19 +
.../pom.xml | 30 +
.../nifi-http-context-map-api/pom.xml | 37 +
.../org/apache/nifi/http/HttpContextMap.java | 69 +
.../nifi-http-context-map-nar/pom.xml | 37 +
.../src/main/resources/META-INF/NOTICE | 19 +
.../nifi-http-context-map/pom.xml | 44 +
.../nifi/http/StandardHttpContextMap.java | 188 +
...org.apache.nifi.controller.ControllerService | 15 +
.../nifi-http-context-map-bundle/pom.xml | 31 +
.../nifi-load-distribution-service-api/pom.xml | 30 +
.../nifi/loading/LoadDistributionListener.java | 24 +
.../nifi/loading/LoadDistributionService.java | 33 +
.../nifi-ssl-context-nar/pom.xml | 35 +
.../src/main/resources/META-INF/NOTICE | 19 +
.../nifi-ssl-context-service/pom.xml | 47 +
.../nifi/ssl/StandardSSLContextService.java | 446 ++
...org.apache.nifi.controller.ControllerService | 15 +
.../apache/nifi/ssl/SSLContextServiceTest.java | 163 +
.../java/org/apache/nifi/ssl/TestProcessor.java | 47 +
.../src/test/resources/localhost-ks.jks | Bin 0 -> 1298 bytes
.../src/test/resources/localhost-ts.jks | Bin 0 -> 891 bytes
.../nifi-ssl-context-bundle/pom.xml | 28 +
.../nifi-ssl-context-service-api/pom.xml | 30 +
.../org/apache/nifi/ssl/SSLContextService.java | 61 +
.../nifi-standard-services-api-nar/pom.xml | 51 +
nifi-nar-bundles/nifi-standard-services/pom.xml | 37 +
.../nifi-update-attribute-model/pom.xml | 26 +
.../apache/nifi/update/attributes/Action.java | 55 +
.../nifi/update/attributes/Condition.java | 49 +
.../apache/nifi/update/attributes/Criteria.java | 141 +
.../nifi/update/attributes/FlowFilePolicy.java | 34 +
.../org/apache/nifi/update/attributes/Rule.java | 65 +
.../update/attributes/serde/CriteriaSerDe.java | 128 +
.../nifi-update-attribute-nar/pom.xml | 43 +
.../src/main/resources/META-INF/NOTICE | 19 +
.../nifi-update-attribute-processor/pom.xml | 53 +
.../processors/attributes/UpdateAttribute.java | 487 ++
.../org.apache.nifi.processor.Processor | 15 +
.../additionalDetails.html | 253 ++
.../update/attributes/TestUpdateAttribute.java | 425 ++
.../nb-configuration.xml | 18 +
.../nifi-update-attribute-ui/pom.xml | 95 +
.../attributes/UpdateAttributeModelFactory.java | 127 +
.../attributes/api/ObjectMapperResolver.java | 51 +
.../update/attributes/api/RuleResource.java | 687 +++
.../nifi/update/attributes/dto/ActionDTO.java | 68 +
.../update/attributes/dto/ConditionDTO.java | 59 +
.../nifi/update/attributes/dto/DtoFactory.java | 68 +
.../nifi/update/attributes/dto/RuleDTO.java | 65 +
.../update/attributes/entity/ActionEntity.java | 65 +
.../attributes/entity/ConditionEntity.java | 65 +
.../entity/EvaluationContextEntity.java | 74 +
.../update/attributes/entity/RuleEntity.java | 65 +
.../update/attributes/entity/RulesEntity.java | 66 +
.../src/main/resources/META-INF/LICENSE | 238 +
.../src/main/resources/META-INF/NOTICE | 49 +
.../META-INF/nifi-processor-configuration | 15 +
.../src/main/webapp/WEB-INF/jsp/worksheet.jsp | 188 +
.../src/main/webapp/WEB-INF/web.xml | 46 +
.../src/main/webapp/css/main.css | 426 ++
.../src/main/webapp/images/bgInputText.png | Bin 0 -> 139 bytes
.../src/main/webapp/images/buttonNew.png | Bin 0 -> 590 bytes
.../src/main/webapp/images/iconDelete.png | Bin 0 -> 670 bytes
.../src/main/webapp/images/iconInfo.png | Bin 0 -> 550 bytes
.../src/main/webapp/js/application.js | 1910 ++++++++
.../nifi-update-attribute-bundle/pom.xml | 52 +
nifi-nar-bundles/pom.xml | 130 +
nifi-nar-maven-plugin/DISCLAIMER | 15 -
nifi-nar-maven-plugin/LICENSE | 202 -
nifi-nar-maven-plugin/NOTICE | 5 -
nifi-nar-maven-plugin/README.md | 77 -
nifi-nar-maven-plugin/pom.xml | 104 -
.../src/main/java/org/apache/nifi/NarMojo.java | 613 ---
.../resources/META-INF/plexus/components.xml | 52 -
nifi-parent/DISCLAIMER | 15 -
nifi-parent/LICENSE | 202 -
nifi-parent/NOTICE | 5 -
nifi-parent/README.md | 73 -
nifi-parent/pom.xml | 438 --
nifi-site/Gruntfile.js | 316 --
nifi-site/LICENSE | 202 -
nifi-site/README.md | 213 -
nifi-site/bower.json | 19 -
nifi-site/config.rb | 24 -
nifi-site/package.json | 29 -
nifi-site/robots.txt | 4 -
nifi-site/src/images/bgNifiLogo.png | Bin 4232 -> 0 bytes
nifi-site/src/images/feather-small.gif | Bin 7129 -> 0 bytes
nifi-site/src/images/flow-th.png | Bin 144319 -> 0 bytes
nifi-site/src/images/flow.png | Bin 947350 -> 0 bytes
.../src/images/niFi-logo-horizontal-scaled.png | Bin 4462 -> 0 bytes
nifi-site/src/images/niFi-logo-horizontal.png | Bin 13161 -> 0 bytes
nifi-site/src/images/nifi16.ico | Bin 1150 -> 0 bytes
nifi-site/src/images/nifiDrop.svg | 217 -
nifi-site/src/includes/disclaimer.hbs | 13 -
nifi-site/src/includes/footer.hbs | 6 -
nifi-site/src/includes/header.hbs | 22 -
nifi-site/src/includes/topbar.hbs | 72 -
nifi-site/src/js/app.js | 17 -
nifi-site/src/layouts/html.hbs | 3 -
nifi-site/src/layouts/markdown.hbs | 12 -
nifi-site/src/pages/html/developer-guide.hbs | 7 -
nifi-site/src/pages/html/docs.hbs | 7 -
nifi-site/src/pages/html/download.hbs | 86 -
nifi-site/src/pages/html/faq.hbs | 33 -
nifi-site/src/pages/html/index.hbs | 77 -
nifi-site/src/pages/html/mailing_lists.hbs | 77 -
nifi-site/src/pages/html/people.hbs | 104 -
nifi-site/src/pages/html/screencasts.hbs | 87 -
nifi-site/src/pages/markdown/licensing-guide.md | 119 -
nifi-site/src/pages/markdown/quickstart.md | 163 -
nifi-site/src/pages/markdown/release-guide.md | 291 --
nifi-site/src/scss/_settings.scss | 1465 ------
nifi-site/src/scss/app.scss | 209 -
nifi/LICENSE | 479 --
nifi/NOTICE | 6 -
nifi/README.md | 159 -
nifi/nifi-api/.gitignore | 2 -
nifi/nifi-api/pom.xml | 25 -
.../annotation/behavior/DynamicProperties.java | 37 -
.../annotation/behavior/DynamicProperty.java | 46 -
.../behavior/DynamicRelationship.java | 45 -
.../nifi/annotation/behavior/EventDriven.java | 48 -
.../annotation/behavior/ReadsAttribute.java | 41 -
.../annotation/behavior/ReadsAttributes.java | 39 -
.../annotation/behavior/SideEffectFree.java | 45 -
.../annotation/behavior/SupportsBatching.java | 51 -
.../annotation/behavior/TriggerSerially.java | 38 -
.../TriggerWhenAnyDestinationAvailable.java | 40 -
.../annotation/behavior/TriggerWhenEmpty.java | 41 -
.../annotation/behavior/WritesAttribute.java | 41 -
.../annotation/behavior/WritesAttributes.java | 39 -
.../documentation/CapabilityDescription.java | 41 -
.../nifi/annotation/documentation/SeeAlso.java | 44 -
.../nifi/annotation/documentation/Tags.java | 42 -
.../nifi/annotation/lifecycle/OnAdded.java | 52 -
.../nifi/annotation/lifecycle/OnDisabled.java | 62 -
.../nifi/annotation/lifecycle/OnEnabled.java | 69 -
.../nifi/annotation/lifecycle/OnRemoved.java | 55 -
.../nifi/annotation/lifecycle/OnScheduled.java | 67 -
.../nifi/annotation/lifecycle/OnShutdown.java | 53 -
.../nifi/annotation/lifecycle/OnStopped.java | 67 -
.../annotation/lifecycle/OnUnscheduled.java | 59 -
.../notification/OnPrimaryNodeStateChange.java | 44 -
.../notification/PrimaryNodeState.java | 33 -
.../apache/nifi/authorization/Authority.java | 93 -
.../nifi/authorization/AuthorityProvider.java | 182 -
.../AuthorityProviderConfigurationContext.java | 48 -
.../AuthorityProviderInitializationContext.java | 27 -
.../authorization/AuthorityProviderLookup.java | 25 -
.../authorization/DownloadAuthorization.java | 83 -
.../annotation/AuthorityProviderContext.java | 35 -
.../exception/AuthorityAccessException.java | 33 -
.../IdentityAlreadyExistsException.java | 32 -
.../exception/ProviderCreationException.java | 39 -
.../exception/ProviderDestructionException.java | 39 -
.../exception/UnknownIdentityException.java | 32 -
.../AbstractConfigurableComponent.java | 220 -
.../apache/nifi/components/AllowableValue.java | 128 -
.../nifi/components/ConfigurableComponent.java | 73 -
.../nifi/components/PropertyDescriptor.java | 556 ---
.../apache/nifi/components/PropertyValue.java | 189 -
.../nifi/components/ValidationContext.java | 95 -
.../nifi/components/ValidationResult.java | 172 -
.../org/apache/nifi/components/Validator.java | 54 -
.../controller/AbstractControllerService.java | 96 -
.../nifi/controller/ConfigurationContext.java | 61 -
.../nifi/controller/ControllerService.java | 180 -
.../ControllerServiceInitializationContext.java | 40 -
.../controller/ControllerServiceLookup.java | 70 -
.../apache/nifi/controller/FlowFileQueue.java | 203 -
.../apache/nifi/controller/ScheduledState.java | 37 -
.../org/apache/nifi/controller/Snippet.java | 89 -
.../org/apache/nifi/controller/Triggerable.java | 117 -
.../controller/annotation/OnConfigured.java | 44 -
.../repository/ContentRepository.java | 303 --
.../controller/repository/FlowFileRecord.java | 48 -
.../repository/FlowFileRepository.java | 123 -
.../repository/FlowFileSwapManager.java | 63 -
.../controller/repository/QueueProvider.java | 33 -
.../controller/repository/RepositoryRecord.java | 82 -
.../repository/RepositoryRecordType.java | 25 -
.../repository/claim/ContentClaim.java | 52 -
.../repository/claim/ContentClaimManager.java | 142 -
.../controller/status/ConnectionStatus.java | 192 -
.../nifi/controller/status/PortStatus.java | 201 -
.../controller/status/ProcessGroupStatus.java | 584 ---
.../nifi/controller/status/ProcessorStatus.java | 274 --
.../status/RemoteProcessGroupStatus.java | 217 -
.../nifi/controller/status/RunStatus.java | 28 -
.../controller/status/TransmissionStatus.java | 23 -
.../history/ComponentStatusRepository.java | 149 -
.../status/history/MetricDescriptor.java | 66 -
.../status/history/StatusHistory.java | 44 -
.../status/history/StatusSnapshot.java | 42 -
.../controller/status/history/ValueMapper.java | 23 -
.../controller/status/history/ValueReducer.java | 25 -
.../org/apache/nifi/events/EventReporter.java | 29 -
.../nifi/expression/AttributeExpression.java | 75 -
.../expression/AttributeValueDecorator.java | 29 -
.../expression/ExpressionLanguageCompiler.java | 69 -
.../java/org/apache/nifi/flowfile/FlowFile.java | 107 -
.../nifi/flowfile/FlowFilePrioritizer.java | 29 -
.../org/apache/nifi/logging/ComponentLog.java | 104 -
.../java/org/apache/nifi/logging/LogLevel.java | 27 -
.../org/apache/nifi/logging/ProcessorLog.java | 30 -
.../nifi/processor/AbstractProcessor.java | 37 -
.../AbstractSessionFactoryProcessor.java | 117 -
.../org/apache/nifi/processor/DataUnit.java | 245 -
.../apache/nifi/processor/FlowFileFilter.java | 93 -
.../apache/nifi/processor/ProcessContext.java | 129 -
.../apache/nifi/processor/ProcessSession.java | 736 ---
.../nifi/processor/ProcessSessionFactory.java | 30 -
.../org/apache/nifi/processor/Processor.java | 90 -
.../ProcessorInitializationContext.java | 47 -
.../org/apache/nifi/processor/QueueSize.java | 48 -
.../org/apache/nifi/processor/Relationship.java | 126 -
.../nifi/processor/SchedulingContext.java | 70 -
.../annotation/CapabilityDescription.java | 42 -
.../nifi/processor/annotation/EventDriven.java | 51 -
.../nifi/processor/annotation/OnAdded.java | 43 -
.../nifi/processor/annotation/OnRemoved.java | 44 -
.../nifi/processor/annotation/OnScheduled.java | 46 -
.../nifi/processor/annotation/OnShutdown.java | 40 -
.../nifi/processor/annotation/OnStopped.java | 58 -
.../processor/annotation/OnUnscheduled.java | 48 -
.../processor/annotation/SideEffectFree.java | 47 -
.../processor/annotation/SupportsBatching.java | 53 -
.../apache/nifi/processor/annotation/Tags.java | 43 -
.../processor/annotation/TriggerSerially.java | 41 -
.../TriggerWhenAnyDestinationAvailable.java | 43 -
.../processor/annotation/TriggerWhenEmpty.java | 43 -
.../exception/FlowFileAccessException.java | 36 -
.../exception/FlowFileHandlingException.java | 38 -
.../exception/MissingFlowFileException.java | 36 -
.../processor/exception/ProcessException.java | 43 -
.../nifi/processor/io/InputStreamCallback.java | 37 -
.../nifi/processor/io/OutputStreamCallback.java | 38 -
.../nifi/processor/io/StreamCallback.java | 37 -
.../nifi/provenance/ProvenanceEventBuilder.java | 308 --
.../nifi/provenance/ProvenanceEventRecord.java | 235 -
.../provenance/ProvenanceEventRepository.java | 180 -
.../nifi/provenance/ProvenanceEventType.java | 92 -
.../nifi/provenance/ProvenanceReporter.java | 616 ---
.../lineage/ComputeLineageResult.java | 58 -
.../lineage/ComputeLineageSubmission.java | 70 -
.../apache/nifi/provenance/lineage/Lineage.java | 37 -
.../lineage/LineageComputationType.java | 27 -
.../nifi/provenance/lineage/LineageEdge.java | 26 -
.../nifi/provenance/lineage/LineageNode.java | 52 -
.../provenance/lineage/LineageNodeType.java | 23 -
.../lineage/ProvenanceEventLineageNode.java | 32 -
.../apache/nifi/provenance/search/Query.java | 96 -
.../nifi/provenance/search/QueryResult.java | 63 -
.../nifi/provenance/search/QuerySubmission.java | 55 -
.../nifi/provenance/search/SearchTerm.java | 24 -
.../nifi/provenance/search/SearchTerms.java | 39 -
.../nifi/provenance/search/SearchableField.java | 52 -
.../provenance/search/SearchableFieldType.java | 28 -
.../apache/nifi/remote/RemoteDestination.java | 50 -
.../nifi/reporting/AbstractReportingTask.java | 96 -
.../org/apache/nifi/reporting/Bulletin.java | 129 -
.../apache/nifi/reporting/BulletinQuery.java | 121 -
.../nifi/reporting/BulletinRepository.java | 82 -
.../apache/nifi/reporting/ComponentType.java | 58 -
.../org/apache/nifi/reporting/EventAccess.java | 49 -
.../nifi/reporting/InitializationException.java | 32 -
.../apache/nifi/reporting/ReportingContext.java | 89 -
.../ReportingInitializationContext.java | 77 -
.../apache/nifi/reporting/ReportingTask.java | 75 -
.../org/apache/nifi/reporting/Severity.java | 24 -
.../nifi/scheduling/SchedulingStrategy.java | 86 -
.../org/apache/nifi/search/SearchContext.java | 49 -
.../org/apache/nifi/search/SearchResult.java | 78 -
.../java/org/apache/nifi/search/Searchable.java | 27 -
.../nifi/web/ClusterRequestException.java | 39 -
.../org/apache/nifi/web/ComponentDetails.java | 144 -
.../apache/nifi/web/ConfigurationAction.java | 125 -
.../nifi/web/InvalidRevisionException.java | 33 -
.../nifi/web/NiFiWebConfigurationContext.java | 100 -
.../web/NiFiWebConfigurationRequestContext.java | 32 -
.../org/apache/nifi/web/NiFiWebContext.java | 116 -
.../apache/nifi/web/NiFiWebContextConfig.java | 55 -
.../apache/nifi/web/NiFiWebRequestContext.java | 56 -
.../nifi/web/ProcessorConfigurationAction.java | 126 -
.../java/org/apache/nifi/web/ProcessorInfo.java | 111 -
.../nifi/web/ResourceNotFoundException.java | 32 -
.../main/java/org/apache/nifi/web/Revision.java | 81 -
.../org/apache/nifi/web/UiExtensionType.java | 32 -
.../org/apache/nifi/web/ViewableContent.java | 65 -
.../nifi/components/TestPropertyDescriptor.java | 59 -
.../org/apache/nifi/processor/TestDataUnit.java | 44 -
.../src/test/resources/logback-test.xml | 27 -
nifi/nifi-assembly/LICENSE | 915 ----
nifi/nifi-assembly/NOTICE | 788 ----
nifi/nifi-assembly/README.md | 110 -
nifi/nifi-assembly/pom.xml | 501 ---
.../src/main/assembly/dependencies.xml | 140 -
nifi/nifi-bootstrap/pom.xml | 32 -
.../apache/nifi/bootstrap/BootstrapCodec.java | 92 -
.../org/apache/nifi/bootstrap/NiFiListener.java | 141 -
.../java/org/apache/nifi/bootstrap/RunNiFi.java | 1008 -----
.../org/apache/nifi/bootstrap/ShutdownHook.java | 92 -
.../exception/InvalidCommandException.java | 38 -
.../bootstrap/util/LimitingInputStream.java | 107 -
.../nifi-data-provenance-utils/.gitignore | 2 -
.../nifi-data-provenance-utils/pom.xml | 34 -
.../nifi/provenance/AsyncLineageSubmission.java | 88 -
.../nifi/provenance/AsyncQuerySubmission.java | 78 -
.../nifi/provenance/NamedSearchableField.java | 95 -
.../nifi/provenance/SearchableFieldParser.java | 53 -
.../nifi/provenance/SearchableFields.java | 91 -
.../nifi/provenance/StandardLineageResult.java | 326 --
.../StandardProvenanceEventRecord.java | 751 ----
.../nifi/provenance/StandardQueryResult.java | 168 -
.../nifi/provenance/lineage/EdgeNode.java | 74 -
.../nifi/provenance/lineage/EventNode.java | 111 -
.../provenance/lineage/FlowFileLineage.java | 76 -
.../nifi/provenance/lineage/FlowFileNode.java | 84 -
.../nifi-expression-language/pom.xml | 56 -
.../language/antlr/AttributeExpressionLexer.g | 211 -
.../language/antlr/AttributeExpressionParser.g | 140 -
.../output/AttributeExpressionLexer.tokens | 88 -
.../expression/language/EmptyPreparedQuery.java | 62 -
.../language/InvalidPreparedQuery.java | 72 -
.../expression/language/PreparedQuery.java | 39 -
.../attribute/expression/language/Query.java | 1313 ------
.../language/StandardAttributeExpression.java | 65 -
.../StandardExpressionLanguageCompiler.java | 58 -
.../language/StandardPreparedQuery.java | 83 -
.../language/evaluation/BooleanEvaluator.java | 43 -
.../language/evaluation/BooleanQueryResult.java | 43 -
.../language/evaluation/DateEvaluator.java | 45 -
.../language/evaluation/DateQueryResult.java | 45 -
.../language/evaluation/Evaluator.java | 44 -
.../language/evaluation/NumberEvaluator.java | 43 -
.../language/evaluation/NumberQueryResult.java | 43 -
.../language/evaluation/QueryResult.java | 26 -
.../language/evaluation/StringEvaluator.java | 43 -
.../language/evaluation/StringQueryResult.java | 43 -
.../evaluation/cast/BooleanCastEvaluator.java | 50 -
.../evaluation/cast/DateCastEvaluator.java | 117 -
.../evaluation/cast/NumberCastEvaluator.java | 72 -
.../evaluation/cast/StringCastEvaluator.java | 49 -
.../evaluation/functions/AndEvaluator.java | 60 -
.../evaluation/functions/AppendEvaluator.java | 50 -
.../functions/AttributeEvaluator.java | 45 -
.../evaluation/functions/ContainsEvaluator.java | 52 -
.../functions/DateToNumberEvaluator.java | 50 -
.../evaluation/functions/DivideEvaluator.java | 57 -
.../evaluation/functions/EndsWithEvaluator.java | 52 -
.../evaluation/functions/EqualsEvaluator.java | 89 -
.../functions/EqualsIgnoreCaseEvaluator.java | 60 -
.../evaluation/functions/FindEvaluator.java | 71 -
.../evaluation/functions/FormatEvaluator.java | 61 -
.../functions/GreaterThanEvaluator.java | 56 -
.../functions/GreaterThanOrEqualEvaluator.java | 56 -
.../evaluation/functions/HostnameEvaluator.java | 58 -
.../evaluation/functions/IPEvaluator.java | 46 -
.../evaluation/functions/IndexOfEvaluator.java | 52 -
.../evaluation/functions/IsEmptyEvaluator.java | 44 -
.../evaluation/functions/IsNullEvaluator.java | 45 -
.../functions/LastIndexOfEvaluator.java | 52 -
.../evaluation/functions/LengthEvaluator.java | 45 -
.../evaluation/functions/LessThanEvaluator.java | 56 -
.../functions/LessThanOrEqualEvaluator.java | 56 -
.../evaluation/functions/MatchesEvaluator.java | 70 -
.../evaluation/functions/MinusEvaluator.java | 57 -
.../evaluation/functions/ModEvaluator.java | 57 -
.../evaluation/functions/MultiplyEvaluator.java | 57 -
.../evaluation/functions/NotEvaluator.java | 49 -
.../evaluation/functions/NotNullEvaluator.java | 45 -
.../evaluation/functions/NowEvaluator.java | 39 -
.../functions/NumberToDateEvaluator.java | 51 -
.../functions/OneUpSequenceEvaluator.java | 41 -
.../evaluation/functions/OrEvaluator.java | 60 -
.../evaluation/functions/PlusEvaluator.java | 57 -
.../evaluation/functions/PrependEvaluator.java | 50 -
.../functions/ReplaceAllEvaluator.java | 55 -
.../functions/ReplaceEmptyEvaluator.java | 51 -
.../evaluation/functions/ReplaceEvaluator.java | 55 -
.../functions/ReplaceNullEvaluator.java | 47 -
.../functions/StartsWithEvaluator.java | 52 -
.../functions/StringToDateEvaluator.java | 64 -
.../functions/SubstringAfterEvaluator.java | 59 -
.../functions/SubstringAfterLastEvaluator.java | 55 -
.../functions/SubstringBeforeEvaluator.java | 58 -
.../functions/SubstringBeforeLastEvaluator.java | 55 -
.../functions/SubstringEvaluator.java | 64 -
.../evaluation/functions/ToLowerEvaluator.java | 45 -
.../evaluation/functions/ToNumberEvaluator.java | 45 -
.../evaluation/functions/ToRadixEvaluator.java | 76 -
.../evaluation/functions/ToStringEvaluator.java | 45 -
.../evaluation/functions/ToUpperEvaluator.java | 45 -
.../evaluation/functions/TrimEvaluator.java | 45 -
.../functions/UrlDecodeEvaluator.java | 55 -
.../functions/UrlEncodeEvaluator.java | 55 -
.../evaluation/functions/UuidEvaluator.java | 39 -
.../literals/BooleanLiteralEvaluator.java | 44 -
.../literals/NumberLiteralEvaluator.java | 44 -
.../literals/StringLiteralEvaluator.java | 77 -
.../evaluation/literals/ToLiteralEvaluator.java | 43 -
.../evaluation/reduce/CountEvaluator.java | 56 -
.../evaluation/reduce/JoinEvaluator.java | 60 -
.../evaluation/reduce/ReduceEvaluator.java | 23 -
.../selection/AllAttributesEvaluator.java | 73 -
.../selection/AnyAttributeEvaluator.java | 73 -
.../selection/DelineatedAttributeEvaluator.java | 87 -
.../selection/IteratingEvaluator.java | 33 -
.../evaluation/selection/MappingEvaluator.java | 71 -
.../selection/MultiAttributeEvaluator.java | 23 -
.../selection/MultiMatchAttributeEvaluator.java | 87 -
.../selection/MultiNamedAttributeEvaluator.java | 69 -
.../AttributeExpressionLanguageException.java | 34 -
...ibuteExpressionLanguageParsingException.java | 34 -
.../exception/IllegalAttributeException.java | 30 -
.../expression/language/TestQuery.java | 1176 -----
.../language/TestStandardPreparedQuery.java | 92 -
.../nifi-commons/nifi-flowfile-packager/pom.xml | 34 -
.../org/apache/nifi/util/FlowFilePackager.java | 28 -
.../apache/nifi/util/FlowFilePackagerV1.java | 104 -
.../apache/nifi/util/FlowFilePackagerV2.java | 146 -
.../apache/nifi/util/FlowFilePackagerV3.java | 93 -
.../apache/nifi/util/FlowFileUnpackager.java | 30 -
.../apache/nifi/util/FlowFileUnpackagerV1.java | 155 -
.../apache/nifi/util/FlowFileUnpackagerV2.java | 143 -
.../apache/nifi/util/FlowFileUnpackagerV3.java | 161 -
.../java/org/apache/nifi/util/Unpackage.java | 119 -
.../nifi/util/TestPackageUnpackageV3.java | 56 -
.../nifi-hl7-query-language/.gitignore | 3 -
.../nifi-hl7-query-language/pom.xml | 127 -
.../apache/nifi/hl7/query/antlr/HL7QueryLexer.g | 173 -
.../nifi/hl7/query/antlr/HL7QueryParser.g | 108 -
.../org/apache/nifi/hl7/hapi/EmptyField.java | 37 -
.../org/apache/nifi/hl7/hapi/HapiField.java | 84 -
.../org/apache/nifi/hl7/hapi/HapiMessage.java | 95 -
.../org/apache/nifi/hl7/hapi/HapiSegment.java | 69 -
.../apache/nifi/hl7/hapi/SingleValueField.java | 43 -
.../java/org/apache/nifi/hl7/io/HL7Reader.java | 27 -
.../hl7/io/exception/InvalidHL7Exception.java | 41 -
.../org/apache/nifi/hl7/model/HL7Component.java | 26 -
.../org/apache/nifi/hl7/model/HL7Field.java | 20 -
.../org/apache/nifi/hl7/model/HL7Message.java | 27 -
.../org/apache/nifi/hl7/model/HL7Segment.java | 27 -
.../org/apache/nifi/hl7/query/Declaration.java | 29 -
.../org/apache/nifi/hl7/query/HL7Query.java | 384 --
.../org/apache/nifi/hl7/query/QueryResult.java | 30 -
.../org/apache/nifi/hl7/query/ResultHit.java | 26 -
.../org/apache/nifi/hl7/query/Selection.java | 38 -
.../hl7/query/evaluator/BooleanEvaluator.java | 24 -
.../nifi/hl7/query/evaluator/Evaluator.java | 28 -
.../hl7/query/evaluator/IntegerEvaluator.java | 25 -
.../hl7/query/evaluator/StringEvaluator.java | 25 -
.../comparison/AbstractComparisonEvaluator.java | 111 -
.../comparison/AbstractNumericComparison.java | 68 -
.../evaluator/comparison/EqualsEvaluator.java | 32 -
.../comparison/GreaterThanEvaluator.java | 32 -
.../comparison/GreaterThanOrEqualEvaluator.java | 32 -
.../evaluator/comparison/IsNullEvaluator.java | 70 -
.../evaluator/comparison/LessThanEvaluator.java | 32 -
.../comparison/LessThanOrEqualEvaluator.java | 32 -
.../comparison/NotEqualsEvaluator.java | 32 -
.../evaluator/comparison/NotEvaluator.java | 37 -
.../evaluator/comparison/NotNullEvaluator.java | 66 -
.../literal/IntegerLiteralEvaluator.java | 36 -
.../literal/StringLiteralEvaluator.java | 36 -
.../hl7/query/evaluator/logic/AndEvaluator.java | 44 -
.../hl7/query/evaluator/logic/OrEvaluator.java | 44 -
.../message/DeclaredReferenceEvaluator.java | 43 -
.../query/evaluator/message/DotEvaluator.java | 89 -
.../query/evaluator/message/FieldEvaluator.java | 68 -
.../evaluator/message/MessageEvaluator.java | 34 -
.../evaluator/message/SegmentEvaluator.java | 52 -
.../exception/HL7QueryParsingException.java | 38 -
.../nifi/hl7/query/result/MissedResult.java | 57 -
.../hl7/query/result/StandardQueryResult.java | 70 -
.../hl7/query/result/StandardResultHit.java | 42 -
.../org/apache/nifi/hl7/query/TestHL7Query.java | 306 --
.../src/test/resources/hyperglycemia | 5 -
.../src/test/resources/hypoglycemia | 5 -
nifi/nifi-commons/nifi-logging-utils/pom.xml | 31 -
.../java/org/apache/nifi/logging/NiFiLog.java | 366 --
.../nifi-processor-utilities/pom.xml | 40 -
.../nifi/processor/util/FlowFileFilters.java | 65 -
.../nifi/processor/util/SSLProperties.java | 236 -
.../nifi/processor/util/StandardValidators.java | 708 ---
.../apache/nifi/processor/TestFormatUtils.java | 40 -
.../processor/util/TestStandardValidators.java | 89 -
nifi/nifi-commons/nifi-properties/.gitignore | 3 -
nifi/nifi-commons/nifi-properties/pom.xml | 24 -
.../org/apache/nifi/util/NiFiProperties.java | 932 ----
.../java/org/apache/nifi/util/StringUtils.java | 66 -
.../apache/nifi/util/NiFiPropertiesTest.java | 127 -
.../NiFiProperties/conf/nifi.blank.properties | 128 -
.../NiFiProperties/conf/nifi.missing.properties | 126 -
.../NiFiProperties/conf/nifi.properties | 128 -
nifi/nifi-commons/nifi-security-utils/pom.xml | 35 -
.../nifi/security/util/CertificateUtils.java | 159 -
.../nifi/security/util/EncryptionMethod.java | 86 -
.../apache/nifi/security/util/KeystoreType.java | 26 -
.../nifi/security/util/SecurityStoreTypes.java | 144 -
.../nifi/security/util/SslContextFactory.java | 182 -
.../nifi-site-to-site-client/pom.xml | 54 -
.../remote/AbstractCommunicationsSession.java | 55 -
.../org/apache/nifi/remote/Communicant.java | 46 -
.../main/java/org/apache/nifi/remote/Peer.java | 151 -
.../org/apache/nifi/remote/PeerDescription.java | 80 -
.../java/org/apache/nifi/remote/PeerStatus.java | 61 -
.../nifi/remote/RemoteResourceInitiator.java | 75 -
.../org/apache/nifi/remote/Transaction.java | 225 -
.../nifi/remote/TransactionCompletion.java | 64 -
.../apache/nifi/remote/TransferDirection.java | 33 -
.../nifi/remote/VersionedRemoteResource.java | 24 -
.../apache/nifi/remote/client/KeystoreType.java | 24 -
.../nifi/remote/client/SiteToSiteClient.java | 771 ----
.../remote/client/SiteToSiteClientConfig.java | 148 -
.../client/socket/EndpointConnection.java | 55 -
.../client/socket/EndpointConnectionPool.java | 994 -----
.../nifi/remote/client/socket/SocketClient.java | 224 -
.../remote/cluster/AdaptedNodeInformation.java | 66 -
.../remote/cluster/ClusterNodeInformation.java | 67 -
.../nifi/remote/cluster/NodeInformation.java | 98 -
.../remote/cluster/NodeInformationAdapter.java | 41 -
.../apache/nifi/remote/codec/FlowFileCodec.java | 72 -
.../remote/codec/StandardFlowFileCodec.java | 128 -
.../remote/exception/HandshakeException.java | 38 -
.../exception/PortNotRunningException.java | 31 -
.../remote/exception/ProtocolException.java | 40 -
.../remote/exception/UnknownPortException.java | 31 -
.../SocketChannelCommunicationsSession.java | 111 -
.../remote/io/socket/SocketChannelInput.java | 72 -
.../remote/io/socket/SocketChannelOutput.java | 59 -
.../SSLSocketChannelCommunicationsSession.java | 114 -
.../io/socket/ssl/SSLSocketChannelInput.java | 56 -
.../io/socket/ssl/SSLSocketChannelOutput.java | 45 -
.../nifi/remote/protocol/ClientProtocol.java | 76 -
.../remote/protocol/CommunicationsInput.java | 34 -
.../remote/protocol/CommunicationsOutput.java | 27 -
.../remote/protocol/CommunicationsSession.java | 62 -
.../apache/nifi/remote/protocol/DataPacket.java | 48 -
.../nifi/remote/protocol/RequestType.java | 43 -
.../protocol/socket/HandshakeProperty.java | 59 -
.../nifi/remote/protocol/socket/Response.java | 52 -
.../remote/protocol/socket/ResponseCode.java | 148 -
.../protocol/socket/SocketClientProtocol.java | 446 --
.../socket/SocketClientTransaction.java | 406 --
.../SocketClientTransactionCompletion.java | 57 -
.../nifi/remote/util/NiFiRestApiUtil.java | 100 -
.../nifi/remote/util/PeerStatusCache.java | 44 -
.../nifi/remote/util/StandardDataPacket.java | 50 -
.../socket/TestEndpointConnectionStatePool.java | 92 -
.../client/socket/TestSiteToSiteClient.java | 103 -
nifi/nifi-commons/nifi-socket-utils/pom.xml | 55 -
.../nifi/io/nio/AbstractChannelReader.java | 162 -
.../java/org/apache/nifi/io/nio/BufferPool.java | 110 -
.../apache/nifi/io/nio/ChannelDispatcher.java | 158 -
.../org/apache/nifi/io/nio/ChannelListener.java | 227 -
.../nifi/io/nio/DatagramChannelReader.java | 59 -
.../apache/nifi/io/nio/SocketChannelReader.java | 51 -
.../io/nio/consumer/AbstractStreamConsumer.java | 131 -
.../nifi/io/nio/consumer/StreamConsumer.java | 80 -
.../io/nio/consumer/StreamConsumerFactory.java | 26 -
.../nifi/io/socket/SSLContextFactory.java | 102 -
.../io/socket/ServerSocketConfiguration.java | 81 -
.../nifi/io/socket/SocketConfiguration.java | 114 -
.../apache/nifi/io/socket/SocketListener.java | 210 -
.../org/apache/nifi/io/socket/SocketUtils.java | 167 -
.../socket/multicast/DiscoverableService.java | 42 -
.../multicast/DiscoverableServiceImpl.java | 77 -
.../multicast/MulticastConfiguration.java | 98 -
.../io/socket/multicast/MulticastListener.java | 192 -
.../multicast/MulticastServiceDiscovery.java | 33 -
.../multicast/MulticastServicesBroadcaster.java | 32 -
.../socket/multicast/MulticastTimeToLive.java | 49 -
.../io/socket/multicast/MulticastUtils.java | 108 -
.../io/socket/multicast/ServiceDiscovery.java | 30 -
.../socket/multicast/ServicesBroadcaster.java | 55 -
.../apache/nifi/io/nio/example/ServerMain.java | 140 -
.../apache/nifi/io/nio/example/TCPClient.java | 86 -
.../apache/nifi/io/nio/example/UDPClient.java | 50 -
.../io/nio/example/UselessStreamConsumer.java | 42 -
.../src/test/resources/log4j.xml | 36 -
nifi/nifi-commons/nifi-utils/.gitignore | 8 -
nifi/nifi-commons/nifi-utils/pom.xml | 30 -
.../flowfile/attributes/CoreAttributes.java | 65 -
.../attributes/FlowFileAttributeKey.java | 22 -
.../nifi/remote/StandardVersionNegotiator.java | 81 -
.../apache/nifi/remote/VersionNegotiator.java | 60 -
.../TransmissionDisabledException.java | 24 -
.../nifi/remote/io/CompressionInputStream.java | 185 -
.../nifi/remote/io/CompressionOutputStream.java | 146 -
.../remote/io/InterruptableInputStream.java | 117 -
.../remote/io/InterruptableOutputStream.java | 81 -
.../remote/io/socket/BufferStateManager.java | 109 -
.../io/socket/SocketChannelInputStream.java | 170 -
.../io/socket/SocketChannelOutputStream.java | 114 -
.../remote/io/socket/ssl/SSLSocketChannel.java | 614 ---
.../socket/ssl/SSLSocketChannelInputStream.java | 65 -
.../ssl/SSLSocketChannelOutputStream.java | 52 -
.../nifi/stream/io/BufferedInputStream.java | 34 -
.../nifi/stream/io/BufferedOutputStream.java | 129 -
.../nifi/stream/io/ByteArrayInputStream.java | 205 -
.../nifi/stream/io/ByteArrayOutputStream.java | 220 -
.../nifi/stream/io/ByteCountingInputStream.java | 110 -
.../stream/io/ByteCountingOutputStream.java | 66 -
.../apache/nifi/stream/io/DataOutputStream.java | 370 --
.../apache/nifi/stream/io/GZIPOutputStream.java | 39 -
.../stream/io/LeakyBucketStreamThrottler.java | 330 --
.../nifi/stream/io/LimitingInputStream.java | 111 -
.../stream/io/MinimumLengthInputStream.java | 91 -
.../nifi/stream/io/NonCloseableInputStream.java | 55 -
.../stream/io/NonCloseableOutputStream.java | 51 -
.../apache/nifi/stream/io/NullOutputStream.java | 46 -
.../apache/nifi/stream/io/StreamThrottler.java | 33 -
.../org/apache/nifi/stream/io/StreamUtils.java | 245 -
.../apache/nifi/stream/io/ZipOutputStream.java | 37 -
.../exception/BytePatternNotFoundException.java | 28 -
.../io/util/NonThreadSafeCircularBuffer.java | 69 -
.../org/apache/nifi/util/BooleanHolder.java | 25 -
.../java/org/apache/nifi/util/EscapeUtils.java | 41 -
.../java/org/apache/nifi/util/FormatUtils.java | 205 -
.../org/apache/nifi/util/IntegerHolder.java | 54 -
.../java/org/apache/nifi/util/LongHolder.java | 58 -
.../apache/nifi/util/NaiveSearchRingBuffer.java | 122 -
.../java/org/apache/nifi/util/ObjectHolder.java | 38 -
.../java/org/apache/nifi/util/RingBuffer.java | 281 --
.../java/org/apache/nifi/util/StopWatch.java | 126 -
.../main/java/org/apache/nifi/util/Tuple.java | 77 -
.../concurrency/DebugDisabledTimedLock.java | 64 -
.../util/concurrency/DebugEnabledTimedLock.java | 130 -
.../util/concurrency/DebuggableTimedLock.java | 30 -
.../apache/nifi/util/concurrency/TimedLock.java | 59 -
.../org/apache/nifi/util/file/FileUtils.java | 594 ---
.../file/monitor/CompoundUpdateMonitor.java | 112 -
.../util/file/monitor/LastModifiedMonitor.java | 30 -
.../nifi/util/file/monitor/MD5SumMonitor.java | 51 -
.../file/monitor/SynchronousFileWatcher.java | 121 -
.../nifi/util/file/monitor/UpdateMonitor.java | 25 -
.../org/apache/nifi/util/search/Search.java | 49 -
.../org/apache/nifi/util/search/SearchTerm.java | 138 -
.../util/search/ahocorasick/AhoCorasick.java | 155 -
.../nifi/util/search/ahocorasick/Node.java | 71 -
.../util/search/ahocorasick/SearchState.java | 63 -
.../nifi/util/timebuffer/EntityAccess.java | 26 -
.../nifi/util/timebuffer/LongEntityAccess.java | 43 -
.../nifi/util/timebuffer/TimedBuffer.java | 114 -
.../nifi/util/timebuffer/TimestampedLong.java | 35 -
.../io/TestCompressionInputOutputStreams.java | 153 -
.../stream/io/ByteCountingInputStreamTest.java | 55 -
.../stream/io/TestLeakyBucketThrottler.java | 147 -
.../nifi/util/TestNaiveSearchRingBuffer.java | 72 -
.../file/monitor/TestCompoundUpdateMonitor.java | 70 -
.../monitor/TestSynchronousFileWatcher.java | 60 -
.../nifi/util/timebuffer/TestRingBuffer.java | 192 -
.../nifi/util/timebuffer/TestTimedBuffer.java | 106 -
.../src/test/resources/logback-test.xml | 32 -
nifi/nifi-commons/nifi-web-utils/pom.xml | 51 -
.../org/apache/nifi/web/util/ClientUtils.java | 132 -
.../nifi/web/util/ObjectMapperResolver.java | 48 -
.../java/org/apache/nifi/web/util/WebUtils.java | 197 -
.../nifi-write-ahead-log/.gitignore | 2 -
nifi/nifi-commons/nifi-write-ahead-log/pom.xml | 35 -
.../org/wali/MinimalLockingWriteAheadLog.java | 1028 -----
.../src/main/java/org/wali/SerDe.java | 128 -
.../src/main/java/org/wali/SyncListener.java | 62 -
.../src/main/java/org/wali/UpdateType.java | 49 -
.../java/org/wali/WriteAheadRepository.java | 122 -
.../src/test/java/org/wali/DummyRecord.java | 66 -
.../test/java/org/wali/DummyRecordSerde.java | 115 -
.../wali/TestMinimalLockingWriteAheadLog.java | 513 ---
nifi/nifi-commons/pom.xml | 41 -
nifi/nifi-docs/LICENSE | 235 -
nifi/nifi-docs/NOTICE | 5 -
nifi/nifi-docs/pom.xml | 152 -
.../src/main/asciidoc/administration-guide.adoc | 692 ---
.../src/main/asciidoc/asciidoc-mod.css | 418 --
.../src/main/asciidoc/developer-guide.adoc | 2170 ---------
.../asciidoc/expression-language-guide.adoc | 1745 --------
.../images/add-controller-service-window.png | Bin 157241 -> 0 bytes
.../images/add-processor-with-tag-cloud.png | Bin 22925 -> 0 bytes
.../src/main/asciidoc/images/add-processor.png | Bin 31524 -> 0 bytes
.../src/main/asciidoc/images/addConnect.png | Bin 1996 -> 0 bytes
.../src/main/asciidoc/images/comments-tab.png | Bin 6431 -> 0 bytes
.../src/main/asciidoc/images/components.png | Bin 8992 -> 0 bytes
.../configure-controller-service-properties.png | Bin 107830 -> 0 bytes
.../configure-controller-service-settings.png | Bin 79718 -> 0 bytes
.../configure-reporting-task-properties.png | Bin 58292 -> 0 bytes
.../configure-reporting-task-settings.png | Bin 80213 -> 0 bytes
.../asciidoc/images/connection-settings.png | Bin 15214 -> 0 bytes
.../images/controller-services-edit-buttons.png | Bin 22294 -> 0 bytes
.../controller-services-edit-buttons2.png | Bin 21845 -> 0 bytes
.../asciidoc/images/controller-services-tab.png | Bin 167278 -> 0 bytes
.../images/controller-settings-button.png | Bin 32341 -> 0 bytes
.../main/asciidoc/images/create-connection.png | Bin 9430 -> 0 bytes
.../images/create-service-ssl-context.png | Bin 142764 -> 0 bytes
.../asciidoc/images/edit-property-dropdown.png | Bin 14291 -> 0 bytes
.../asciidoc/images/edit-property-textarea.png | Bin 17146 -> 0 bytes
.../main/asciidoc/images/event-attributes.png | Bin 94668 -> 0 bytes
.../src/main/asciidoc/images/event-content.png | Bin 88726 -> 0 bytes
.../src/main/asciidoc/images/event-details.png | Bin 139736 -> 0 bytes
.../src/main/asciidoc/images/expand-event.png | Bin 40476 -> 0 bytes
.../main/asciidoc/images/expanded-events.png | Bin 76082 -> 0 bytes
.../src/main/asciidoc/images/find-parents.png | Bin 35831 -> 0 bytes
.../src/main/asciidoc/images/iconAlert.png | Bin 1396 -> 0 bytes
.../src/main/asciidoc/images/iconConnection.png | Bin 1517 -> 0 bytes
.../src/main/asciidoc/images/iconDelete.png | Bin 670 -> 0 bytes
.../src/main/asciidoc/images/iconDisable.png | Bin 764 -> 0 bytes
.../src/main/asciidoc/images/iconEdit.png | Bin 493 -> 0 bytes
.../src/main/asciidoc/images/iconEnable.png | Bin 667 -> 0 bytes
.../src/main/asciidoc/images/iconExport.png | Bin 453 -> 0 bytes
.../main/asciidoc/images/iconFlowHistory.png | Bin 2463 -> 0 bytes
.../src/main/asciidoc/images/iconFunnel.png | Bin 1223 -> 0 bytes
.../src/main/asciidoc/images/iconInfo.png | Bin 550 -> 0 bytes
.../src/main/asciidoc/images/iconInputPort.png | Bin 1842 -> 0 bytes
.../main/asciidoc/images/iconInputPortSmall.png | Bin 532 -> 0 bytes
.../src/main/asciidoc/images/iconLabel.png | Bin 838 -> 0 bytes
.../src/main/asciidoc/images/iconLineage.png | Bin 2214 -> 0 bytes
.../main/asciidoc/images/iconNewTemplate.png | Bin 557 -> 0 bytes
.../src/main/asciidoc/images/iconNotSecure.png | Bin 221 -> 0 bytes
.../src/main/asciidoc/images/iconOutputPort.png | Bin 1658 -> 0 bytes
.../asciidoc/images/iconOutputPortSmall.png | Bin 459 -> 0 bytes
.../main/asciidoc/images/iconProcessGroup.png | Bin 1422 -> 0 bytes
.../src/main/asciidoc/images/iconProcessor.png | Bin 1446 -> 0 bytes
.../src/main/asciidoc/images/iconProvenance.png | Bin 2268 -> 0 bytes
.../asciidoc/images/iconRemoteProcessGroup.png | Bin 674 -> 0 bytes
.../src/main/asciidoc/images/iconResize.png | Bin 165 -> 0 bytes
.../src/main/asciidoc/images/iconRun.png | Bin 538 -> 0 bytes
.../src/main/asciidoc/images/iconSecure.png | Bin 225 -> 0 bytes
.../src/main/asciidoc/images/iconSettings.png | Bin 2638 -> 0 bytes
.../src/main/asciidoc/images/iconStop.png | Bin 402 -> 0 bytes
.../src/main/asciidoc/images/iconSummary.png | Bin 272 -> 0 bytes
.../src/main/asciidoc/images/iconTemplate.png | Bin 970 -> 0 bytes
.../asciidoc/images/iconTransmissionActive.png | Bin 1330 -> 0 bytes
.../images/iconTransmissionInactive.png | Bin 1248 -> 0 bytes
.../src/main/asciidoc/images/iconUsers.png | Bin 2272 -> 0 bytes
.../main/asciidoc/images/iconViewDetails.png | Bin 1788 -> 0 bytes
.../images/instantiate-template-description.png | Bin 6508 -> 0 bytes
.../asciidoc/images/instantiate-template.png | Bin 3216 -> 0 bytes
.../main/asciidoc/images/invalid-processor.png | Bin 8187 -> 0 bytes
.../main/asciidoc/images/lineage-flowfile.png | Bin 3855 -> 0 bytes
.../asciidoc/images/lineage-graph-annotated.png | Bin 170122 -> 0 bytes
nifi/nifi-docs/src/main/asciidoc/images/ncm.png | Bin 339522 -> 0 bytes
.../src/main/asciidoc/images/new-flow.png | Bin 266913 -> 0 bytes
.../main/asciidoc/images/nifi-arch-cluster.png | Bin 50384 -> 0 bytes
.../src/main/asciidoc/images/nifi-arch.png | Bin 43509 -> 0 bytes
.../main/asciidoc/images/nifi-navigation.png | Bin 339195 -> 0 bytes
.../asciidoc/images/nifi-toolbar-components.png | Bin 262314 -> 0 bytes
.../images/nifi_first_launch_screenshot.png | Bin 122540 -> 0 bytes
.../src/main/asciidoc/images/parent-found.png | Bin 42814 -> 0 bytes
.../asciidoc/images/process-group-anatomy.png | Bin 48554 -> 0 bytes
.../main/asciidoc/images/processor-anatomy.png | Bin 37035 -> 0 bytes
.../images/processor-connection-bubble.png | Bin 9936 -> 0 bytes
.../src/main/asciidoc/images/properties-tab.png | Bin 11847 -> 0 bytes
.../asciidoc/images/provenance-annotated.png | Bin 530929 -> 0 bytes
.../main/asciidoc/images/provenance-table.png | Bin 466599 -> 0 bytes
.../asciidoc/images/remote-group-anatomy.png | Bin 61383 -> 0 bytes
.../images/remote-group-ports-dialog.png | Bin 21662 -> 0 bytes
.../images/remote-port-connection-status.png | Bin 57332 -> 0 bytes
.../images/reporting-tasks-edit-buttons.png | Bin 5049 -> 0 bytes
.../images/reporting-tasks-edit-buttons2.png | Bin 17525 -> 0 bytes
.../asciidoc/images/reporting-tasks-tab.png | Bin 161551 -> 0 bytes
.../src/main/asciidoc/images/scheduling-tab.png | Bin 12288 -> 0 bytes
.../src/main/asciidoc/images/search-events.png | Bin 62626 -> 0 bytes
.../images/search-receive-event-abc.png | Bin 67817 -> 0 bytes
.../asciidoc/images/settings-general-tab.png | Bin 132657 -> 0 bytes
.../src/main/asciidoc/images/settings-tab.png | Bin 16225 -> 0 bytes
.../src/main/asciidoc/images/simple-flow.png | Bin 196808 -> 0 bytes
.../src/main/asciidoc/images/stats-history.png | Bin 33205 -> 0 bytes
.../src/main/asciidoc/images/status-bar.png | Bin 249077 -> 0 bytes
.../main/asciidoc/images/summary-annotated.png | Bin 111956 -> 0 bytes
.../src/main/asciidoc/images/summary-table.png | Bin 62114 -> 0 bytes
.../main/asciidoc/images/valid-processor.png | Bin 8090 -> 0 bytes
nifi/nifi-docs/src/main/asciidoc/overview.adoc | 296 --
.../nifi-docs/src/main/asciidoc/user-guide.adoc | 1468 ------
.../src/main/assembly/dependencies.xml | 44 -
nifi/nifi-external/README.md | 19 -
.../nifi-nifi-example-nar/pom.xml | 35 -
.../nifi-nifi-example-processors/pom.xml | 70 -
.../nifi/processors/WriteResourceToStream.java | 102 -
.../org.apache.nifi.processor.Processor | 15 -
.../src/main/resources/file.txt | 1 -
.../processors/WriteResourceToStreamTest.java | 47 -
nifi/nifi-external/nifi-example-bundle/pom.xml | 42 -
nifi/nifi-external/nifi-spark-receiver/pom.xml | 36 -
.../org/apache/nifi/spark/NiFiDataPacket.java | 39 -
.../org/apache/nifi/spark/NiFiReceiver.java | 208 -
nifi/nifi-external/nifi-storm-spout/pom.xml | 38 -
.../org/apache/nifi/storm/NiFiDataPacket.java | 39 -
.../java/org/apache/nifi/storm/NiFiSpout.java | 266 --
nifi/nifi-external/pom.xml | 32 -
.../nifi-processor-bundle-archetype/pom.xml | 47 -
.../META-INF/maven/archetype-metadata.xml | 43 -
.../nifi-__artifactBaseName__-nar/pom.xml | 37 -
.../pom.xml | 54 -
.../src/main/java/MyProcessor.java | 96 -
.../org.apache.nifi.processor.Processor | 15 -
.../src/test/java/MyProcessorTest.java | 39 -
.../main/resources/archetype-resources/pom.xml | 35 -
.../nifi-service-bundle-archetype/pom.xml | 48 -
.../META-INF/maven/archetype-metadata.xml | 55 -
.../nifi-__artifactBaseName__-api-nar/pom.xml | 42 -
.../nifi-__artifactBaseName__-api/pom.xml | 35 -
.../src/main/java/MyService.java | 30 -
.../nifi-__artifactBaseName__-nar/pom.xml | 43 -
.../nifi-__artifactBaseName__/pom.xml | 59 -
.../src/main/java/StandardMyService.java | 80 -
...org.apache.nifi.controller.ControllerService | 15 -
.../src/test/java/TestProcessor.java | 45 -
.../src/test/java/TestStandardMyService.java | 45 -
.../main/resources/archetype-resources/pom.xml | 37 -
nifi/nifi-maven-archetypes/pom.xml | 30 -
nifi/nifi-mock/pom.xml | 49 -
.../MockProvenanceEventRepository.java | 131 -
.../apache/nifi/reporting/BulletinFactory.java | 43 -
.../org/apache/nifi/reporting/MockBulletin.java | 24 -
.../util/ControllerServiceConfiguration.java | 74 -
.../nifi/util/MockBulletinRepository.java | 74 -
.../nifi/util/MockConfigurationContext.java | 77 -
...kControllerServiceInitializationContext.java | 58 -
.../nifi/util/MockControllerServiceLookup.java | 100 -
.../org/apache/nifi/util/MockEventAccess.java | 70 -
.../java/org/apache/nifi/util/MockFlowFile.java | 277 --
.../org/apache/nifi/util/MockFlowFileQueue.java | 85 -
.../apache/nifi/util/MockProcessContext.java | 286 --
.../apache/nifi/util/MockProcessSession.java | 1037 -----
.../MockProcessorInitializationContext.java | 84 -
.../org/apache/nifi/util/MockProcessorLog.java | 276 --
.../org/apache/nifi/util/MockPropertyValue.java | 185 -
.../nifi/util/MockProvenanceReporter.java | 453 --
.../apache/nifi/util/MockReportingContext.java | 115 -
.../MockReportingInitializationContext.java | 89 -
.../apache/nifi/util/MockSessionFactory.java | 48 -
.../apache/nifi/util/MockValidationContext.java | 135 -
.../org/apache/nifi/util/ReflectionUtils.java | 150 -
.../apache/nifi/util/SharedSessionState.java | 91 -
.../apache/nifi/util/SingleSessionFactory.java | 35 -
.../nifi/util/StandardProcessorTestRunner.java | 728 ---
.../java/org/apache/nifi/util/TestRunner.java | 765 ----
.../java/org/apache/nifi/util/TestRunners.java | 37 -
.../CurrentTestStandardProcessorTestRunner.java | 71 -
.../util/TestStandardProcessorTestRunner.java | 90 -
.../nifi-aws-bundle/nifi-aws-nar/pom.xml | 36 -
.../src/main/resources/META-INF/NOTICE | 74 -
.../nifi-aws-bundle/nifi-aws-processors/pom.xml | 67 -
.../processors/aws/AbstractAWSProcessor.java | 190 -
.../processors/aws/s3/AbstractS3Processor.java | 169 -
.../nifi/processors/aws/s3/FetchS3Object.java | 155 -
.../nifi/processors/aws/s3/PutS3Object.java | 181 -
.../aws/sns/AbstractSNSProcessor.java | 58 -
.../apache/nifi/processors/aws/sns/PutSNS.java | 155 -
.../aws/sqs/AbstractSQSProcessor.java | 51 -
.../nifi/processors/aws/sqs/DeleteSQS.java | 95 -
.../apache/nifi/processors/aws/sqs/GetSQS.java | 205 -
.../apache/nifi/processors/aws/sqs/PutSQS.java | 146 -
.../org.apache.nifi.processor.Processor | 20 -
.../processors/aws/s3/TestFetchS3Object.java | 61 -
.../nifi/processors/aws/s3/TestPutS3Object.java | 102 -
.../nifi/processors/aws/sns/TestPutSNS.java | 51 -
.../nifi/processors/aws/sqs/TestGetSQS.java | 49 -
.../nifi/processors/aws/sqs/TestPutSQS.java | 52 -
.../src/test/resources/hello.txt | 1 -
nifi/nifi-nar-bundles/nifi-aws-bundle/pom.xml | 43 -
.../nifi-flume-bundle/nifi-flume-nar/pom.xml | 132 -
.../src/main/resources/META-INF/LICENSE | 319 --
.../src/main/resources/META-INF/NOTICE | 192 -
.../nifi-flume-processors/pom.xml | 178 -
.../flume/AbstractFlumeProcessor.java | 160 -
.../nifi/processors/flume/ExecuteFlumeSink.java | 144 -
.../processors/flume/ExecuteFlumeSource.java | 195 -
.../processors/flume/NifiChannelSelector.java | 69 -
.../processors/flume/NifiSessionChannel.java | 47 -
.../flume/NifiSessionFactoryChannel.java | 50 -
.../flume/NifiSinkSessionChannel.java | 49 -
.../processors/flume/NifiSinkTransaction.java | 71 -
.../nifi/processors/flume/NifiTransaction.java | 54 -
.../processors/flume/util/FlowFileEvent.java | 134 -
.../flume/util/FlowFileEventConstants.java | 40 -
.../org.apache.nifi.processor.Processor | 16 -
.../additionalDetails.html | 159 -
.../additionalDetails.html | 114 -
.../processors/flume/ExecuteFlumeSinkTest.java | 169 -
.../flume/ExecuteFlumeSourceTest.java | 151 -
.../src/test/resources/core-site-broken.xml | 25 -
.../src/test/resources/core-site.xml | 30 -
.../src/test/resources/log4j.properties | 22 -
.../src/test/resources/simplelogger.properties | 21 -
.../src/test/resources/testdata/records.txt | 4 -
nifi/nifi-nar-bundles/nifi-flume-bundle/pom.xml | 40 -
.../nifi-framework-nar/.gitignore | 1 -
.../nifi-framework-nar/pom.xml | 62 -
.../src/main/resources/META-INF/LICENSE | 264 --
.../src/main/resources/META-INF/NOTICE | 137 -
.../nifi-administration/.gitignore | 1 -
.../nifi-framework/nifi-administration/pom.xml | 121 -
.../nifi/admin/AuditDataSourceFactoryBean.java | 222 -
.../org/apache/nifi/admin/RepositoryUtils.java | 90 -
.../nifi/admin/UserDataSourceFactoryBean.java | 235 -
.../org/apache/nifi/admin/dao/ActionDAO.java | 72 -
.../org/apache/nifi/admin/dao/AuthorityDAO.java | 59 -
.../org/apache/nifi/admin/dao/DAOFactory.java | 29 -
.../nifi/admin/dao/DataAccessException.java | 39 -
.../java/org/apache/nifi/admin/dao/UserDAO.java | 127 -
.../nifi/admin/dao/impl/DAOFactoryImpl.java | 51 -
.../nifi/admin/dao/impl/StandardActionDAO.java | 965 ----
.../admin/dao/impl/StandardAuthorityDAO.java | 172 -
.../nifi/admin/dao/impl/StandardUserDAO.java | 634 ---
.../admin/service/AccountDisabledException.java | 40 -
.../admin/service/AccountNotFoundException.java | 40 -
.../admin/service/AccountPendingException.java | 41 -
.../admin/service/AdministrationException.java | 39 -
.../apache/nifi/admin/service/AuditService.java | 74 -
.../apache/nifi/admin/service/UserService.java | 157 -
.../service/action/AbstractUserAction.java | 97 -
.../admin/service/action/AddActionsAction.java | 48 -
.../service/action/AdministrationAction.java | 38 -
.../service/action/AuthorizeDownloadAction.java | 54 -
.../service/action/AuthorizeUserAction.java | 173 -
.../admin/service/action/CreateUserAction.java | 53 -
.../admin/service/action/DeleteUserAction.java | 68 -
.../admin/service/action/DisableUserAction.java | 76 -
.../service/action/DisableUserGroupAction.java | 69 -
.../service/action/FindUserByDnAction.java | 49 -
.../service/action/FindUserByIdAction.java | 46 -
.../admin/service/action/GetActionAction.java | 41 -
.../admin/service/action/GetActionsAction.java | 48 -
.../admin/service/action/GetPreviousValues.java | 43 -
.../service/action/GetUserGroupAction.java | 50 -
.../admin/service/action/GetUsersAction.java | 39 -
.../service/action/HasPendingUserAccounts.java | 34 -
.../action/InvalidateUserAccountAction.java | 58 -
.../InvalidateUserGroupAccountsAction.java | 45 -
.../service/action/PurgeActionsAction.java | 51 -
.../action/RequestUserAccountAction.java | 67 -
.../service/action/SeedUserAccountsAction.java | 164 -
.../admin/service/action/UngroupUserAction.java | 69 -
.../service/action/UngroupUserGroupAction.java | 57 -
.../admin/service/action/UpdateUserAction.java | 124 -
.../UpdateUserAuthoritiesCacheAction.java | 73 -
.../service/action/UpdateUserCacheAction.java | 47 -
.../service/action/UpdateUserGroupAction.java | 171 -
.../service/impl/StandardAuditService.java | 219 -
.../admin/service/impl/StandardUserService.java | 642 ---
.../admin/service/transaction/Transaction.java | 49 -
.../service/transaction/TransactionBuilder.java | 25 -
.../transaction/TransactionException.java | 40 -
.../transaction/impl/StandardTransaction.java | 93 -
.../impl/StandardTransactionBuilder.java | 57 -
.../AuthorityProviderFactoryBean.java | 490 --
...rdAuthorityProviderConfigurationContext.java | 50 -
...dAuthorityProviderInitializationContext.java | 42 -
.../java/org/apache/nifi/history/History.java | 56 -
.../org/apache/nifi/history/HistoryQuery.java | 99 -
.../org/apache/nifi/history/PreviousValue.java | 54 -
.../org/apache/nifi/user/AccountStatus.java | 47 -
.../java/org/apache/nifi/user/NiFiUser.java | 164 -
.../org/apache/nifi/user/NiFiUserGroup.java | 45 -
.../resources/nifi-administration-context.xml | 62 -
.../src/main/xsd/authority-providers.xsd | 49 -
.../service/action/AuthorizeUserActionTest.java | 433 --
.../service/action/CreateUserActionTest.java | 144 -
.../service/action/DisableUserActionTest.java | 168 -
.../action/InvalidateUserAccountActionTest.java | 126 -
.../action/RequestUserAccountActionTest.java | 127 -
.../action/SeedUserAccountsActionTest.java | 262 --
.../action/SetUserAuthoritiesActionTest.java | 223 -
.../nifi-framework/nifi-client-dto/.gitignore | 6 -
.../nifi-framework/nifi-client-dto/pom.xml | 30 -
.../org/apache/nifi/web/api/dto/AboutDTO.java | 64 -
.../org/apache/nifi/web/api/dto/BannerDTO.java | 64 -
.../nifi/web/api/dto/BulletinBoardDTO.java | 65 -
.../apache/nifi/web/api/dto/BulletinDTO.java | 168 -
.../nifi/web/api/dto/BulletinQueryDTO.java | 120 -
.../org/apache/nifi/web/api/dto/ClusterDTO.java | 64 -
.../nifi/web/api/dto/ComponentHistoryDTO.java | 59 -
.../apache/nifi/web/api/dto/ConnectableDTO.java | 157 -
.../apache/nifi/web/api/dto/ConnectionDTO.java | 237 -
.../web/api/dto/ControllerConfigurationDTO.java | 187 -
.../apache/nifi/web/api/dto/ControllerDTO.java | 277 --
.../nifi/web/api/dto/ControllerServiceDTO.java | 204 -
...ontrollerServiceReferencingComponentDTO.java | 219 -
.../org/apache/nifi/web/api/dto/CounterDTO.java | 101 -
.../apache/nifi/web/api/dto/CountersDTO.java | 64 -
.../nifi/web/api/dto/DocumentedTypeDTO.java | 75 -
.../apache/nifi/web/api/dto/FlowSnippetDTO.java | 165 -
.../org/apache/nifi/web/api/dto/FunnelDTO.java | 29 -
.../org/apache/nifi/web/api/dto/LabelDTO.java | 100 -
.../nifi/web/api/dto/NiFiComponentDTO.java | 106 -
.../org/apache/nifi/web/api/dto/NodeDTO.java | 202 -
.../apache/nifi/web/api/dto/NodeEventDTO.java | 78 -
.../web/api/dto/NodeSystemDiagnosticsDTO.java | 59 -
.../org/apache/nifi/web/api/dto/PortDTO.java | 173 -
.../apache/nifi/web/api/dto/PositionDTO.java | 68 -
.../nifi/web/api/dto/PreviousValueDTO.java | 77 -
.../nifi/web/api/dto/ProcessGroupDTO.java | 236 -
.../nifi/web/api/dto/ProcessorConfigDTO.java | 292 --
.../apache/nifi/web/api/dto/ProcessorDTO.java | 199 -
.../nifi/web/api/dto/PropertyDescriptorDTO.java | 251 --
.../nifi/web/api/dto/PropertyHistoryDTO.java | 45 -
.../nifi/web/api/dto/RelationshipDTO.java | 73 -
.../api/dto/RemoteProcessGroupContentsDTO.java | 59 -
.../nifi/web/api/dto/RemoteProcessGroupDTO.java | 291 --
.../web/api/dto/RemoteProcessGroupPortDTO.java | 202 -
.../nifi/web/api/dto/ReportingTaskDTO.java | 253 --
.../apache/nifi/web/api/dto/RevisionDTO.java | 82 -
.../org/apache/nifi/web/api/dto/SnippetDTO.java | 247 -
.../nifi/web/api/dto/SystemDiagnosticsDTO.java | 491 --
.../apache/nifi/web/api/dto/TemplateDTO.java | 124 -
.../org/apache/nifi/web/api/dto/UserDTO.java | 187 -
.../apache/nifi/web/api/dto/UserGroupDTO.java | 89 -
.../nifi/web/api/dto/action/ActionDTO.java | 187 -
.../nifi/web/api/dto/action/HistoryDTO.java | 78 -
.../web/api/dto/action/HistoryQueryDTO.java | 153 -
.../component/details/ComponentDetailsDTO.java | 32 -
.../component/details/ExtensionDetailsDTO.java | 43 -
.../details/RemoteProcessGroupDetailsDTO.java | 43 -
.../dto/action/details/ActionDetailsDTO.java | 34 -
.../dto/action/details/ConfigureDetailsDTO.java | 73 -
.../dto/action/details/ConnectDetailsDTO.java | 134 -
.../api/dto/action/details/MoveDetailsDTO.java | 88 -
.../api/dto/action/details/PurgeDetailsDTO.java | 47 -
.../web/api/dto/provenance/AttributeDTO.java | 73 -
.../web/api/dto/provenance/ProvenanceDTO.java | 174 -
.../api/dto/provenance/ProvenanceEventDTO.java | 662 ---
.../dto/provenance/ProvenanceOptionsDTO.java | 45 -
.../dto/provenance/ProvenanceRequestDTO.java | 125 -
.../dto/provenance/ProvenanceResultsDTO.java | 144 -
.../ProvenanceSearchableFieldDTO.java | 89 -
.../api/dto/provenance/lineage/LineageDTO.java | 171 -
.../provenance/lineage/LineageRequestDTO.java | 90 -
.../provenance/lineage/LineageResultsDTO.java | 77 -
.../provenance/lineage/ProvenanceLinkDTO.java | 107 -
.../provenance/lineage/ProvenanceNodeDTO.java | 172 -
.../dto/search/ComponentSearchResultDTO.java | 90 -
.../web/api/dto/search/NodeSearchResultDTO.java | 59 -
.../web/api/dto/search/SearchResultsDTO.java | 136 -
.../dto/search/UserGroupSearchResultDTO.java | 44 -
.../web/api/dto/search/UserSearchResultDTO.java | 59 -
.../dto/status/ClusterConnectionStatusDTO.java | 94 -
.../api/dto/status/ClusterPortStatusDTO.java | 94 -
.../status/ClusterProcessGroupStatusDTO.java | 102 -
.../dto/status/ClusterProcessorStatusDTO.java | 127 -
.../ClusterRemoteProcessGroupStatusDTO.java | 94 -
.../web/api/dto/status/ClusterStatusDTO.java | 46 -
.../api/dto/status/ClusterStatusHistoryDTO.java | 79 -
.../web/api/dto/status/ConnectionStatusDTO.java | 211 -
.../web/api/dto/status/ControllerStatusDTO.java | 231 -
.../api/dto/status/NodeConnectionStatusDTO.java | 60 -
.../web/api/dto/status/NodePortStatusDTO.java | 60 -
.../dto/status/NodeProcessGroupStatusDTO.java | 64 -
.../api/dto/status/NodeProcessorStatusDTO.java | 60 -
.../status/NodeRemoteProcessGroupStatusDTO.java | 60 -
.../nifi/web/api/dto/status/NodeStatusDTO.java | 60 -
.../api/dto/status/NodeStatusHistoryDTO.java | 60 -
.../nifi/web/api/dto/status/PortStatusDTO.java | 149 -
.../api/dto/status/ProcessGroupStatusDTO.java | 371 --
.../web/api/dto/status/ProcessorStatusDTO.java | 214 -
.../web/api/dto/status/RemotePortStatusDTO.java | 104 -
.../dto/status/RemoteProcessGroupStatusDTO.java | 167 -
.../nifi/web/api/dto/status/StatusDTO.java | 45 -
.../web/api/dto/status/StatusDescriptorDTO.java | 105 -
.../web/api/dto/status/StatusHistoryDTO.java | 97 -
.../api/dto/status/StatusHistoryDetailDTO.java | 59 -
.../web/api/dto/status/StatusSnapshotDTO.java | 61 -
.../nifi/web/api/dto/util/DateTimeAdapter.java | 47 -
.../nifi/web/api/dto/util/TimeAdapter.java | 47 -
.../nifi/web/api/dto/util/TimestampAdapter.java | 47 -
.../apache/nifi/web/api/entity/AboutEntity.java | 43 -
.../nifi/web/api/entity/ActionEntity.java | 43 -
.../nifi/web/api/entity/AuthorityEntity.java | 54 -
.../nifi/web/api/entity/BannerEntity.java | 44 -
.../web/api/entity/BulletinBoardEntity.java | 43 -
.../entity/ClusterConnectionStatusEntity.java | 43 -
.../nifi/web/api/entity/ClusterEntity.java | 43 -
.../web/api/entity/ClusterPortStatusEntity.java | 43 -
.../entity/ClusterProcessGroupStatusEntity.java | 43 -
.../entity/ClusterProcessorStatusEntity.java | 43 -
.../ClusterRemoteProcessGroupStatusEntity.java | 44 -
.../api/entity/ClusterSearchResultsEntity.java | 42 -
.../web/api/entity/ClusterStatusEntity.java | 43 -
.../api/entity/ClusterStatusHistoryEntity.java | 43 -
.../web/api/entity/ComponentHistoryEntity.java | 43 -
.../nifi/web/api/entity/ConnectionEntity.java | 41 -
.../nifi/web/api/entity/ConnectionsEntity.java | 43 -
.../entity/ControllerConfigurationEntity.java | 43 -
.../nifi/web/api/entity/ControllerEntity.java | 43 -
.../web/api/entity/ControllerServiceEntity.java | 41 -
...ollerServiceReferencingComponentsEntity.java | 43 -
.../entity/ControllerServiceTypesEntity.java | 42 -
.../api/entity/ControllerServicesEntity.java | 42 -
.../web/api/entity/ControllerStatusEntity.java | 43 -
.../nifi/web/api/entity/CounterEntity.java | 41 -
.../nifi/web/api/entity/CountersEntity.java | 41 -
.../org/apache/nifi/web/api/entity/Entity.java | 45 -
.../nifi/web/api/entity/FlowSnippetEntity.java | 43 -
.../nifi/web/api/entity/FunnelEntity.java | 43 -
.../nifi/web/api/entity/FunnelsEntity.java | 43 -
.../nifi/web/api/entity/HistoryEntity.java | 43 -
.../nifi/web/api/entity/InputPortEntity.java | 41 -
.../nifi/web/api/entity/InputPortsEntity.java | 43 -
.../apache/nifi/web/api/entity/LabelEntity.java | 43 -
.../nifi/web/api/entity/LabelsEntity.java | 43 -
.../nifi/web/api/entity/LineageEntity.java | 43 -
.../apache/nifi/web/api/entity/NodeEntity.java | 43 -
.../nifi/web/api/entity/NodeStatusEntity.java | 43 -
.../api/entity/NodeSystemDiagnosticsEntity.java | 43 -
.../nifi/web/api/entity/OutputPortEntity.java | 41 -
.../nifi/web/api/entity/OutputPortsEntity.java | 43 -
.../web/api/entity/PrioritizerTypesEntity.java | 42 -
.../nifi/web/api/entity/ProcessGroupEntity.java | 43 -
.../api/entity/ProcessGroupStatusEntity.java | 43 -
.../web/api/entity/ProcessGroupsEntity.java | 44 -
.../nifi/web/api/entity/ProcessorEntity.java | 43 -
.../web/api/entity/ProcessorTypesEntity.java | 42 -
.../nifi/web/api/entity/ProcessorsEntity.java | 43 -
.../api/entity/PropertyDescriptorEntity.java | 43 -
.../nifi/web/api/entity/ProvenanceEntity.java | 38 -
.../web/api/entity/ProvenanceEventEntity.java | 43 -
.../web/api/entity/ProvenanceOptionsEntity.java | 44 -
.../api/entity/RemoteProcessGroupEntity.java | 43 -
.../entity/RemoteProcessGroupPortEntity.java | 43 -
.../api/entity/RemoteProcessGroupsEntity.java | 43 -
.../web/api/entity/ReportingTaskEntity.java | 41 -
.../api/entity/ReportingTaskTypesEntity.java | 42 -
.../web/api/entity/ReportingTasksEntity.java | 42 -
.../web/api/entity/SearchResultsEntity.java | 42 -
.../nifi/web/api/entity/SnippetEntity.java | 43 -
.../web/api/entity/StatusHistoryEntity.java | 43 -
.../web/api/entity/SystemDiagnosticsEntity.java | 43 -
.../nifi/web/api/entity/TemplateEntity.java | 43 -
.../nifi/web/api/entity/TemplatesEntity.java | 59 -
.../apache/nifi/web/api/entity/UserEntity.java | 43 -
.../nifi/web/api/entity/UserGroupEntity.java | 43 -
.../web/api/entity/UserSearchResultsEntity.java | 56 -
.../apache/nifi/web/api/entity/UsersEntity.java | 60 -
.../.gitignore | 1 -
.../nifi-cluster-authorization-provider/pom.xml | 46 -
.../ClusterManagerAuthorizationProvider.java | 225 -
.../NodeAuthorizationProvider.java | 389 --
.../protocol/message/DoesDnExistMessage.java | 55 -
.../protocol/message/GetAuthoritiesMessage.java | 57 -
.../message/GetGroupForUserMessage.java | 54 -
.../protocol/message/ProtocolMessage.java | 56 -
.../message/jaxb/JaxbProtocolUtils.java | 41 -
.../protocol/message/jaxb/ObjectFactory.java | 44 -
....apache.nifi.authorization.AuthorityProvider | 16 -
.../nifi-framework/nifi-documentation/pom.xml | 44 -
.../ConfigurableComponentInitializer.java | 45 -
.../apache/nifi/documentation/DocGenerator.java | 180 -
.../nifi/documentation/DocumentationWriter.java | 33 -
.../html/HtmlDocumentationWriter.java | 614 ---
.../html/HtmlProcessorDocumentationWriter.java | 256 --
.../init/ControllerServiceInitializer.java | 57 -
.../init/ProcessorInitializer.java | 55 -
.../init/ReportingTaskingInitializer.java | 54 -
.../mock/MockConfigurationContext.java | 48 -
...kControllerServiceInitializationContext.java | 46 -
.../mock/MockControllerServiceLookup.java | 65 -
.../documentation/mock/MockProcessContext.java | 85 -
.../MockProcessorInitializationContext.java | 45 -
.../documentation/mock/MockProcessorLogger.java | 169 -
.../MockReportingInitializationContext.java | 67 -
.../documentation/util/ReflectionUtils.java | 139 -
.../nifi/documentation/DocGeneratorTest.java | 96 -
.../example/ControllerServiceWithLogger.java | 31 -
.../FullyDocumentedControllerService.java | 103 -
.../example/FullyDocumentedProcessor.java | 172 -
.../example/FullyDocumentedReportingTask.java | 94 -
.../documentation/example/NakedProcessor.java | 31 -
.../example/ProcessorWithLogger.java | 37 -
.../example/ReportingTaskWithLogger.java | 36 -
.../documentation/example/SampleService.java | 25 -
.../html/HtmlDocumentationWriterTest.java | 156 -
.../html/ProcessorDocumentationWriterTest.java | 132 -
.../nifi/documentation/html/XmlValidator.java | 53 -
.../src/test/resources/conf/nifi.properties | 129 -
.../src/test/resources/lib/example.nar | Bin 721040 -> 0 bytes
.../src/test/resources/lib/jetty.nar | Bin 4638519 -> 0 bytes
.../test/resources/lib/nifi-framework-nar.nar | Bin 406 -> 0 bytes
.../nifi-file-authorization-provider/pom.xml | 85 -
.../FileAuthorizationProvider.java | 496 ---
....apache.nifi.authorization.AuthorityProvider | 15 -
.../src/main/xsd/users.xsd | 64 -
.../FileAuthorizationProviderTest.java | 128 -
.../nifi-framework-cluster-protocol/.gitignore | 1 -
.../nifi-framework-cluster-protocol/pom.xml | 67 -
.../protocol/ClusterManagerProtocolSender.java | 73 -
.../cluster/protocol/ConnectionRequest.java | 43 -
.../cluster/protocol/ConnectionResponse.java | 139 -
.../apache/nifi/cluster/protocol/Heartbeat.java | 68 -
.../nifi/cluster/protocol/NodeBulletins.java | 44 -
.../nifi/cluster/protocol/NodeIdentifier.java | 183 -
.../cluster/protocol/NodeProtocolSender.java | 84 -
.../nifi/cluster/protocol/ProtocolContext.java | 40 -
.../cluster/protocol/ProtocolException.java | 40 -
.../nifi/cluster/protocol/ProtocolHandler.java | 45 -
.../nifi/cluster/protocol/ProtocolListener.java | 77 -
.../protocol/ProtocolMessageMarshaller.java | 38 -
.../protocol/ProtocolMessageUnmarshaller.java | 39 -
.../nifi/cluster/protocol/StandardDataFlow.java | 98 -
.../UnknownServiceAddressException.java | 39 -
.../impl/ClusterManagerProtocolSenderImpl.java | 246 -
.../ClusterManagerProtocolSenderListener.java | 117 -
.../protocol/impl/ClusterServiceDiscovery.java | 180 -
.../protocol/impl/ClusterServiceLocator.java | 228 -
.../impl/ClusterServicesBroadcaster.java | 185 -
.../protocol/impl/CopyingInputStream.java | 78 -
.../impl/MulticastProtocolListener.java | 203 -
.../protocol/impl/NodeProtocolSenderImpl.java | 170 -
.../impl/NodeProtocolSenderListener.java | 115 -
.../protocol/impl/SocketProtocolListener.java | 204 -
.../protocol/jaxb/JaxbProtocolContext.java | 147 -
.../jaxb/message/AdaptedConnectionRequest.java | 40 -
.../jaxb/message/AdaptedConnectionResponse.java | 109 -
.../protocol/jaxb/message/AdaptedCounter.java | 56 -
.../protocol/jaxb/message/AdaptedDataFlow.java | 64 -
.../protocol/jaxb/message/AdaptedHeartbeat.java | 66 -
.../jaxb/message/AdaptedNodeBulletins.java | 50 -
.../jaxb/message/AdaptedNodeIdentifier.java | 76 -
.../jaxb/message/ConnectionRequestAdapter.java | 40 -
.../jaxb/message/ConnectionResponseAdapter.java | 54 -
.../protocol/jaxb/message/DataFlowAdapter.java | 49 -
.../protocol/jaxb/message/HeartbeatAdapter.java | 53 -
.../jaxb/message/JaxbProtocolUtils.java | 41 -
.../jaxb/message/NodeBulletinsAdapter.java | 47 -
.../jaxb/message/NodeIdentifierAdapter.java | 50 -
.../protocol/jaxb/message/ObjectFactory.java | 104 -
.../message/ConnectionRequestMessage.java | 46 -
.../message/ConnectionResponseMessage.java | 63 -
.../ControllerStartupFailureMessage.java | 49 -
.../protocol/message/DisconnectMessage.java | 54 -
.../protocol/message/ExceptionMessage.java | 44 -
.../protocol/message/FlowRequestMessage.java | 45 -
.../protocol/message/FlowResponseMessage.java | 43 -
.../protocol/message/HeartbeatMessage.java | 42 -
.../message/MulticastProtocolMessage.java | 66 -
.../protocol/message/NodeBulletinsMessage.java | 42 -
.../cluster/protocol/message/PingMessage.java | 55 -
.../message/PrimaryRoleAssignmentMessage.java | 55 -
.../protocol/message/ProtocolMessage.java | 61 -
.../message/ReconnectionFailureMessage.java | 47 -
.../message/ReconnectionRequestMessage.java | 94 -
.../message/ReconnectionResponseMessage.java | 32 -
.../message/ServiceBroadcastMessage.java | 64 -
.../MulticastConfigurationFactoryBean.java | 60 -
.../ServerSocketConfigurationFactoryBean.java | 66 -
.../spring/SocketConfigurationFactoryBean.java | 66 -
.../resources/nifi-cluster-protocol-context.xml | 110 -
.../ClusterManagerProtocolSenderImplTest.java | 133 -
.../impl/ClusterServiceDiscoveryTest.java | 134 -
.../impl/ClusterServiceLocatorTest.java | 121 -
.../impl/ClusterServicesBroadcasterTest.java | 131 -
.../impl/MulticastProtocolListenerTest.java | 168 -
.../impl/NodeProtocolSenderImplTest.java | 198 -
.../impl/SocketProtocolListenerTest.java | 132 -
.../impl/testutils/DelayedProtocolHandler.java | 56 -
.../testutils/ReflexiveProtocolHandler.java | 46 -
.../nifi-framework-cluster-web/.gitignore | 1 -
.../nifi-framework-cluster-web/pom.xml | 44 -
.../nifi/cluster/context/ClusterContext.java | 65 -
.../cluster/context/ClusterContextImpl.java | 69 -
.../context/ClusterContextThreadLocal.java | 42 -
.../nifi-framework-cluster/.gitignore | 1 -
.../nifi-framework-cluster/pom.xml | 152 -
.../cluster/client/MulticastTestClient.java | 151 -
.../org/apache/nifi/cluster/event/Event.java | 119 -
.../apache/nifi/cluster/event/EventManager.java | 62 -
.../cluster/event/impl/EventManagerImpl.java | 140 -
.../cluster/firewall/ClusterNodeFirewall.java | 34 -
.../impl/FileBasedClusterNodeFirewall.java | 207 -
.../nifi/cluster/flow/ClusterDataFlow.java | 55 -
.../apache/nifi/cluster/flow/DaoException.java | 39 -
.../apache/nifi/cluster/flow/DataFlowDao.java | 60 -
.../cluster/flow/DataFlowManagementService.java | 124 -
.../nifi/cluster/flow/PersistedFlowState.java | 36 -
.../nifi/cluster/flow/StaleFlowException.java | 40 -
.../nifi/cluster/flow/impl/DataFlowDaoImpl.java | 597 ---
.../impl/DataFlowManagementServiceImpl.java | 403 --
.../nifi/cluster/manager/ClusterManager.java | 182 -
.../cluster/manager/HttpClusterManager.java | 129 -
.../cluster/manager/HttpRequestReplicator.java | 83 -
.../cluster/manager/HttpResponseMapper.java | 40 -
.../nifi/cluster/manager/NodeResponse.java | 321 --
.../exception/BlockedByFirewallException.java | 60 -
.../manager/exception/ClusterException.java | 39 -
.../ConnectingNodeMutableRequestException.java | 39 -
...DisconnectedNodeMutableRequestException.java | 39 -
.../exception/IllegalClusterStateException.java | 39 -
.../exception/IllegalNodeDeletionException.java | 39 -
.../IllegalNodeDisconnectionException.java | 39 -
.../IllegalNodeReconnectionException.java | 39 -
.../IneligiblePrimaryNodeException.java | 39 -
.../exception/MutableRequestException.java | 40 -
.../exception/NoConnectedNodesException.java | 39 -
.../exception/NoResponseFromNodesException.java | 40 -
.../exception/NodeDisconnectionException.java | 39 -
.../exception/NodeReconnectionException.java | 39 -
.../PrimaryRoleAssignmentException.java | 39 -
.../SafeModeMutableRequestException.java | 39 -
.../manager/exception/UnknownNodeException.java | 39 -
.../exception/UriConstructionException.java | 39 -
.../manager/impl/ClusteredEventAccess.java | 135 -
.../manager/impl/ClusteredReportingContext.java | 209 -
.../manager/impl/HttpRequestReplicatorImpl.java | 515 ---
.../manager/impl/HttpResponseMapperImpl.java | 81 -
.../cluster/manager/impl/WebClusterManager.java | 4212 ------------------
.../java/org/apache/nifi/cluster/node/Node.java | 237 -
...anagerProtocolServiceLocatorFactoryBean.java | 114 -
...FileBasedClusterNodeFirewallFactoryBean.java | 58 -
.../spring/WebClusterManagerFactoryBean.java | 132 -
.../reporting/ClusteredReportingTaskNode.java | 49 -
.../resources/nifi-cluster-manager-context.xml | 128 -
.../event/impl/EventManagerImplTest.java | 119 -
.../impl/FileBasedClusterNodeFirewallTest.java | 135 -
.../impl/DataFlowManagementServiceImplTest.java | 345 --
.../impl/HttpRequestReplicatorImplTest.java | 368 --
.../impl/HttpResponseMapperImplTest.java | 125 -
.../manager/impl/TestWebClusterManager.java | 54 -
.../cluster/manager/testutils/HttpRequest.java | 235 -
.../cluster/manager/testutils/HttpResponse.java | 91 -
.../manager/testutils/HttpResponseAction.java | 57 -
.../cluster/manager/testutils/HttpServer.java | 238 -
.../src/test/resources/logback-test.xml | 48 -
.../apache/nifi/cluster/firewall/impl/empty.txt | 0
.../apache/nifi/cluster/firewall/impl/ips.txt | 12 -
.../nifi-framework-core-api/.gitignore | 2 -
.../nifi-framework-core-api/pom.xml | 54 -
.../apache/nifi/cluster/protocol/DataFlow.java | 41 -
.../apache/nifi/connectable/Connectable.java | 274 --
.../nifi/connectable/ConnectableType.java | 44 -
.../org/apache/nifi/connectable/Connection.java | 78 -
.../org/apache/nifi/connectable/Funnel.java | 24 -
.../java/org/apache/nifi/connectable/Port.java | 31 -
.../org/apache/nifi/connectable/Position.java | 36 -
.../java/org/apache/nifi/connectable/Size.java | 36 -
.../controller/AbstractConfiguredComponent.java | 295 --
.../apache/nifi/controller/AbstractPort.java | 608 ---
.../nifi/controller/ConfiguredComponent.java | 61 -
.../nifi/controller/ContentAvailability.java | 55 -
.../org/apache/nifi/controller/Counter.java | 32 -
.../nifi/controller/EventBasedWorker.java | 32 -
.../org/apache/nifi/controller/Heartbeater.java | 22 -
.../nifi/controller/ProcessScheduler.java | 171 -
.../apache/nifi/controller/ProcessorNode.java | 98 -
.../nifi/controller/ReportingTaskNode.java | 101 -
.../nifi/controller/StandardFlowFileQueue.java | 1104 -----
.../apache/nifi/controller/StandardFunnel.java | 538 ---
.../controller/ValidationContextFactory.java | 31 -
.../org/apache/nifi/controller/WorkerQueue.java | 36 -
.../exception/CommunicationsException.java | 40 -
.../exception/ComponentLifeCycleException.java | 30 -
...ControllerServiceAlreadyExistsException.java | 26 -
...ControllerServiceInstantiationException.java | 38 -
.../ProcessorInstantiationException.java | 27 -
.../org/apache/nifi/controller/label/Label.java | 48 -
.../ReportingTaskInstantiationException.java | 31 -
.../reporting/ReportingTaskProvider.java | 111 -
.../repository/ContentNotFoundException.java | 47 -
.../repository/CounterRepository.java | 34 -
.../controller/repository/FlowFileEvent.java | 54 -
.../repository/FlowFileEventRepository.java | 45 -
.../repository/RepositoryStatusReport.java | 28 -
.../service/ControllerServiceNode.java | 75 -
.../service/ControllerServiceProvider.java | 176 -
.../service/ControllerServiceReference.java | 46 -
.../service/ControllerServiceState.java | 43 -
.../org/apache/nifi/events/BulletinFactory.java | 72 -
.../nifi/events/BulletinProcessingStrategy.java | 27 -
.../apache/nifi/events/ComponentBulletin.java | 30 -
.../org/apache/nifi/events/SystemBulletin.java | 32 -
.../org/apache/nifi/groups/ProcessGroup.java | 681 ---
.../apache/nifi/groups/ProcessGroupCounts.java | 66 -
.../apache/nifi/groups/RemoteProcessGroup.java | 200 -
.../RemoteProcessGroupPortDescriptor.java | 72 -
.../org/apache/nifi/logging/LogMessage.java | 74 -
.../org/apache/nifi/logging/LogObserver.java | 22 -
.../org/apache/nifi/logging/LogRepository.java | 67 -
.../nifi/logging/LogRepositoryFactory.java | 61 -
.../nifi/remote/PortAuthorizationResult.java | 25 -
.../nifi/remote/RemoteAuthorizationState.java | 27 -
.../org/apache/nifi/remote/RemoteGroupPort.java | 43 -
.../org/apache/nifi/remote/RootGroupPort.java | 78 -
.../nifi/remote/cluster/NodeInformant.java | 22 -
.../remote/exception/BadRequestException.java | 30 -
.../exception/NotAuthorizedException.java | 26 -
.../exception/RequestExpiredException.java | 26 -
.../nifi/remote/protocol/ServerProtocol.java | 141 -
.../nifi-framework-core/.gitignore | 1 -
.../nifi-framework/nifi-framework-core/pom.xml | 144 -
.../apache/nifi/cluster/BulletinsPayload.java | 94 -
.../nifi/cluster/ConnectionException.java | 41 -
.../nifi/cluster/DisconnectionException.java | 41 -
.../apache/nifi/cluster/HeartbeatPayload.java | 168 -
.../org/apache/nifi/connectable/LocalPort.java | 171 -
.../nifi/connectable/StandardConnection.java | 349 --
.../nifi/controller/EventDrivenWorkerQueue.java | 329 --
.../nifi/controller/FileSystemSwapManager.java | 769 ----
.../apache/nifi/controller/FlowController.java | 3766 ----------------
.../nifi/controller/FlowFromDOMFactory.java | 458 --
.../controller/FlowSerializationException.java | 46 -
.../apache/nifi/controller/FlowSerializer.java | 40 -
.../FlowSynchronizationException.java | 45 -
.../nifi/controller/FlowSynchronizer.java | 43 -
.../nifi/controller/FlowUnmarshaller.java | 77 -
.../apache/nifi/controller/SnippetManager.java | 96 -
.../apache/nifi/controller/StandardCounter.java | 108 -
.../nifi/controller/StandardFlowSerializer.java | 453 --
.../nifi/controller/StandardFlowService.java | 864 ----
.../controller/StandardFlowSynchronizer.java | 1203 -----
.../nifi/controller/StandardProcessorNode.java | 1257 ------
.../apache/nifi/controller/StandardSnippet.java | 153 -
.../org/apache/nifi/controller/Template.java | 37 -
.../apache/nifi/controller/TemplateManager.java | 523 ---
.../controller/UninheritableFlowException.java | 46 -
.../exception/FlowFileConsumptionException.java | 38 -
.../ReportingTaskInstantiationException.java | 31 -
.../exception/ValidationException.java | 47 -
.../nifi/controller/label/StandardLabel.java | 110 -
.../reporting/AbstractReportingTaskNode.java | 252 --
.../reporting/StandardReportingContext.java | 143 -
.../StandardReportingInitializationContext.java | 115 -
.../reporting/StandardReportingTaskNode.java | 40 -
.../repository/BatchingSessionFactory.java | 242 -
.../repository/ConnectionSwapInfo.java | 58 -
.../repository/FileSystemRepository.java | 1358 ------
.../controller/repository/ProcessContext.java | 279 --
.../repository/ProvenanceEventEnricher.java | 33 -
.../repository/RepositoryPurgeException.java | 44 -
.../repository/RingBufferEventRepository.java | 312 --
.../repository/StandardCounterRepository.java | 109 -
.../repository/StandardFlowFileEvent.java | 237 -
.../repository/StandardFlowFileRecord.java | 308 --
.../repository/StandardProcessSession.java | 2700 -----------
.../StandardProcessSessionFactory.java | 33 -
.../repository/StandardProvenanceReporter.java | 460 --
.../repository/StandardRepositoryRecord.java | 202 -
.../StandardRepositoryStatusReport.java | 84 -
.../repository/VolatileContentRepository.java | 641 ---
.../repository/VolatileFlowFileRepository.java | 107 -
.../WriteAheadFlowFileRepository.java | 845 ----
.../repository/claim/ContentDirection.java | 32 -
.../repository/claim/StandardContentClaim.java | 142 -
.../claim/StandardContentClaimManager.java | 145 -
.../repository/io/ArrayManagedOutputStream.java | 245 -
.../repository/io/ByteCountingInputStream.java | 100 -
.../repository/io/ByteCountingOutputStream.java | 62 -
.../io/DisableOnCloseInputStream.java | 91 -
.../io/DisableOnCloseOutputStream.java | 71 -
.../io/FlowFileAccessInputStream.java | 161 -
.../io/FlowFileAccessOutputStream.java | 92 -
.../repository/io/LimitedInputStream.java | 110 -
.../controller/repository/io/LongHolder.java | 46 -
.../controller/repository/io/MemoryManager.java | 55 -
.../repository/io/SyncOnCloseOutputStream.java | 58 -
.../scheduling/ConnectableProcessContext.java | 193 -
.../scheduling/EventDrivenSchedulingAgent.java | 348 --
.../scheduling/ProcessContextFactory.java | 51 -
.../scheduling/QuartzSchedulingAgent.java | 231 -
.../controller/scheduling/ScheduleState.java | 94 -
.../controller/scheduling/SchedulingAgent.java | 45 -
.../scheduling/StandardProcessScheduler.java | 716 ---
.../scheduling/TimerDrivenSchedulingAgent.java | 237 -
.../service/ControllerServiceLoader.java | 164 -
.../service/StandardConfigurationContext.java | 88 -
...dControllerServiceInitializationContext.java | 82 -
.../service/StandardControllerServiceNode.java | 229 -
.../StandardControllerServiceProvider.java | 628 ---
.../StandardControllerServiceReference.java | 105 -
.../history/StandardMetricDescriptor.java | 109 -
.../status/history/StandardStatusHistory.java | 54 -
.../status/history/StandardStatusSnapshot.java | 78 -
.../status/history/StatusHistoryUtil.java | 107 -
.../VolatileComponentStatusRepository.java | 690 ---
.../tasks/ContinuallyRunConnectableTask.java | 111 -
.../tasks/ContinuallyRunProcessorTask.java | 193 -
.../nifi/controller/tasks/ExpireFlowFiles.java | 112 -
.../controller/tasks/ReportingTaskWrapper.java | 66 -
.../nifi/diagnostics/DiagnosticUtils.java | 27 -
.../nifi/diagnostics/GarbageCollection.java | 60 -
.../apache/nifi/diagnostics/StorageUsage.java | 71 -
.../nifi/diagnostics/SystemDiagnostics.java | 211 -
.../diagnostics/SystemDiagnosticsFactory.java | 132 -
.../nifi/encrypt/EncryptionException.java | 42 -
.../apache/nifi/encrypt/StringEncryptor.java | 148 -
.../java/org/apache/nifi/engine/FlowEngine.java | 127 -
.../events/NodeBulletinProcessingStrategy.java | 66 -
.../nifi/events/VolatileBulletinRepository.java | 305 --
.../events/network/CommunicationsFailure.java | 36 -
.../nifi/events/network/NetworkTransfer.java | 48 -
.../nifi/fingerprint/FingerprintException.java | 40 -
.../nifi/fingerprint/FingerprintFactory.java | 1055 -----
.../nifi/groups/StandardProcessGroup.java | 2045 ---------
.../org/apache/nifi/jaxb/AdaptedBulletin.java | 110 -
.../org/apache/nifi/jaxb/AdaptedCounter.java | 63 -
.../org/apache/nifi/jaxb/BulletinAdapter.java | 59 -
.../org/apache/nifi/jaxb/CounterAdapter.java | 50 -
.../org/apache/nifi/lifecycle/LifeCycle.java | 46 -
.../nifi/lifecycle/LifeCycleException.java | 46 -
.../nifi/lifecycle/LifeCycleStartException.java | 46 -
.../nifi/lifecycle/LifeCycleStopException.java | 46 -
.../logging/ControllerServiceLogObserver.java | 46 -
.../nifi/logging/ProcessorLogObserver.java | 48 -
.../nifi/logging/ReportingTaskLogObserver.java | 45 -
.../repository/StandardLogRepository.java | 173 -
.../nifi/persistence/FlowConfigurationDAO.java | 99 -
.../StandardSnippetDeserializer.java | 42 -
.../persistence/StandardSnippetSerializer.java | 47 -
.../StandardXMLFlowConfigurationDAO.java | 158 -
.../nifi/persistence/TemplateDeserializer.java | 42 -
.../nifi/persistence/TemplateSerializer.java | 47 -
.../nifi/processor/SimpleProcessLogger.java | 302 --
.../nifi/processor/StandardProcessContext.java | 182 -
.../StandardProcessorInitializationContext.java | 49 -
.../nifi/processor/StandardPropertyValue.java | 157 -
.../processor/StandardSchedulingContext.java | 114 -
.../processor/StandardValidationContext.java | 136 -
.../StandardValidationContextFactory.java | 44 -
.../org/apache/nifi/remote/RemoteNiFiUtils.java | 203 -
.../nifi/remote/StandardRemoteProcessGroup.java | 1257 ------
...tandardRemoteProcessGroupPortDescriptor.java | 148 -
.../org/apache/nifi/services/FlowService.java | 118 -
.../nifi/spring/FlowControllerFactoryBean.java | 101 -
.../spring/RingBufferEventRepositoryBean.java | 45 -
.../spring/StandardFlowServiceFactoryBean.java | 96 -
.../apache/nifi/util/ComponentStatusReport.java | 137 -
.../java/org/apache/nifi/util/Connectables.java | 56 -
.../java/org/apache/nifi/util/DomUtils.java | 89 -
.../org/apache/nifi/util/ReflectionUtils.java | 277 --
.../java/org/apache/nifi/util/SnippetUtils.java | 239 -
.../src/main/resources/FlowConfiguration.xsd | 374 --
...nifi.controller.repository.ContentRepository | 16 -
...ifi.controller.repository.FlowFileRepository | 16 -
...fi.controller.repository.FlowFileSwapManager | 15 -
...ler.status.history.ComponentStatusRepository | 15 -
.../src/main/resources/nifi-context.xml | 49 -
.../nifi/cluster/HeartbeatPayloadTest.java | 120 -
.../controller/StandardFlowServiceTest.java | 299 --
.../controller/TestFileSystemSwapManager.java | 104 -
.../repository/TestFileSystemRepository.java | 352 --
.../TestRingBufferEventRepository.java | 138 -
.../repository/TestStandardProcessSession.java | 1322 ------
.../TestStandardProvenanceReporter.java | 62 -
.../TestVolatileContentRepository.java | 183 -
.../TestWriteAheadFlowFileRepository.java | 135 -
.../repository/io/TestLimitedInputStream.java | 122 -
.../controller/repository/util/DiskUtils.java | 77 -
.../TestStandardProcessScheduler.java | 107 -
.../StandardControllerServiceProviderTest.java | 71 -
.../TestStandardControllerServiceProvider.java | 379 --
.../controller/service/mock/DummyProcessor.java | 48 -
.../nifi/controller/service/mock/ServiceA.java | 48 -
.../nifi/controller/service/mock/ServiceB.java | 23 -
.../service/util/TestControllerService.java | 61 -
.../fingerprint/FingerprintFactoryTest.java | 59 -
.../processor/TestStandardPropertyValue.java | 176 -
.../processors/DataGeneratorTestProcessor.java | 85 -
.../StubAttributeLoggerProcessor.java | 111 -
.../StubTerminationFileProcessor.java | 40 -
...org.apache.nifi.controller.ControllerService | 15 -
.../src/test/resources/conf/0bytes.xml | 0
.../test/resources/conf/all-flow-corrupt.xml | 201 -
.../resources/conf/all-flow-inheritable.xml | 196 -
.../resources/conf/all-flow-uninheritable.xml | 202 -
.../src/test/resources/conf/all-flow.xml | 198 -
.../src/test/resources/conf/nifi.properties | 127 -
.../test/resources/conf/only-termination.xml | 37 -
.../src/test/resources/conf/remote-flow.xml | 145 -
.../src/test/resources/conf/standard-flow.xml | 196 -
.../src/test/resources/conf/taskConfig.xml | 17 -
.../test/resources/conf/termination-only.xml | 0
.../src/test/resources/hello.txt | 1 -
.../src/test/resources/logback-test.xml | 36 -
.../test/resources/nifi-with-remote.properties | 127 -
.../src/test/resources/nifi.properties | 127 -
.../test/resources/nifi/fingerprint/flow1a.xml | 160 -
.../test/resources/nifi/fingerprint/flow1b.xml | 160 -
.../test/resources/nifi/fingerprint/flow2.xml | 160 -
.../src/test/resources/old-swap-file.swap | Bin 1730054 -> 0 bytes
.../nifi-framework/nifi-nar-utils/.gitignore | 2 -
.../nifi-framework/nifi-nar-utils/pom.xml | 34 -
.../org/apache/nifi/nar/ExtensionManager.java | 187 -
.../org/apache/nifi/nar/ExtensionMapping.java | 73 -
.../org/apache/nifi/nar/NarClassLoader.java | 227 -
.../org/apache/nifi/nar/NarClassLoaders.java | 293 --
.../java/org/apache/nifi/nar/NarCloseable.java | 44 -
.../nifi/nar/NarThreadContextClassLoader.java | 187 -
.../java/org/apache/nifi/nar/NarUnpacker.java | 396 --
.../java/org/apache/nifi/util/FileUtils.java | 186 -
.../org/apache/nifi/nar/NarUnpackerTest.java | 214 -
.../resources/NarUnpacker/conf/nifi.properties | 129 -
.../resources/NarUnpacker/lib/dummy-one.nar | Bin 1749 -> 0 bytes
.../NarUnpacker/lib/nifi-framework-nar.nar | Bin 406 -> 0 bytes
.../resources/NarUnpacker/lib2/dummy-two.nar | Bin 1751 -> 0 bytes
.../nifi-framework/nifi-resources/LICENSE | 202 -
.../nifi-framework/nifi-resources/NOTICE | 5 -
.../nifi-framework/nifi-resources/pom.xml | 50 -
.../src/main/assembly/dependencies.xml | 52 -
.../src/main/resources/bin/dump-nifi.bat | 47 -
.../src/main/resources/bin/nifi.sh | 219 -
.../src/main/resources/bin/run-nifi.bat | 47 -
.../src/main/resources/bin/status-nifi.bat | 47 -
.../main/resources/conf/authority-providers.xml | 43 -
.../main/resources/conf/authorized-users.xml | 57 -
.../src/main/resources/conf/bootstrap.conf | 57 -
.../src/main/resources/conf/logback.xml | 156 -
.../src/main/resources/conf/nifi.properties | 162 -
.../nifi-framework/nifi-runtime/.gitignore | 2 -
.../nifi-framework/nifi-runtime/pom.xml | 45 -
.../java/org/apache/nifi/BootstrapListener.java | 377 --
.../src/main/java/org/apache/nifi/NiFi.java | 228 -
.../main/java/org/apache/nifi/NiFiServer.java | 31 -
.../apache/nifi/util/LimitingInputStream.java | 107 -
.../nifi-framework/nifi-security/.gitignore | 1 -
.../nifi-framework/nifi-security/pom.xml | 39 -
.../util/SslContextCreationException.java | 40 -
.../security/util/SslContextFactory.java | 121 -
.../framework/security/util/SslException.java | 40 -
.../security/util/SslServerSocketFactory.java | 82 -
...SslServerSocketFactoryCreationException.java | 41 -
.../security/util/SslSocketFactory.java | 92 -
.../util/SslSocketFactoryCreationException.java | 40 -
.../security/util/SslContextFactoryTest.java | 68 -
.../src/test/resources/localhost-ks.jks | Bin 1298 -> 0 bytes
.../src/test/resources/localhost-ts.jks | Bin 891 -> 0 bytes
.../src/test/resources/log4j.properties | 21 -
.../nifi-framework/nifi-site-to-site/.gitignore | 2 -
.../nifi-framework/nifi-site-to-site/pom.xml | 75 -
.../util/RemoteProcessGroupUtils.java | 218 -
.../nifi/remote/RemoteResourceFactory.java | 168 -
.../nifi/remote/RemoteResourceManager.java | 227 -
.../apache/nifi/remote/RemoteSiteListener.java | 33 -
.../nifi/remote/SocketRemoteSiteListener.java | 352 --
.../nifi/remote/StandardRemoteGroupPort.java | 447 --
.../nifi/remote/StandardRootGroupPort.java | 635 ---
.../exception/UnsupportedCodecException.java | 32 -
.../socket/ClusterManagerServerProtocol.java | 207 -
.../socket/SocketFlowFileServerProtocol.java | 642 ---
.../org.apache.nifi.remote.codec.FlowFileCodec | 15 -
...g.apache.nifi.remote.protocol.ClientProtocol | 15 -
...g.apache.nifi.remote.protocol.ServerProtocol | 16 -
.../remote/TestStandardSiteToSiteProtocol.java | 116 -
.../io/socket/TestSocketChannelStreams.java | 231 -
.../io/socket/ssl/TestSSLSocketChannel.java | 381 --
.../test/resources/dummy-certs/localhost-ks.jks | Bin 1298 -> 0 bytes
.../test/resources/dummy-certs/localhost-ts.jks | Bin 891 -> 0 bytes
.../src/test/resources/logback-test.xml | 30 -
.../src/test/resources/nifi.properties | 111 -
.../nifi-framework/nifi-user-actions/.gitignore | 1 -
.../nifi-framework/nifi-user-actions/pom.xml | 24 -
.../java/org/apache/nifi/action/Action.java | 121 -
.../java/org/apache/nifi/action/Component.java | 34 -
.../java/org/apache/nifi/action/Operation.java | 37 -
.../component/details/ComponentDetails.java | 26 -
.../component/details/ExtensionDetails.java | 34 -
.../details/RemoteProcessGroupDetails.java | 34 -
.../nifi/action/details/ActionDetails.java | 26 -
.../nifi/action/details/ConfigureDetails.java | 52 -
.../nifi/action/details/ConnectDetails.java | 90 -
.../apache/nifi/action/details/MoveDetails.java | 61 -
.../nifi/action/details/PurgeDetails.java | 45 -
.../nifi-custom-ui-utilities/.gitignore | 1 -
.../nifi-web/nifi-custom-ui-utilities/pom.xml | 34 -
.../HttpServletConfigurationRequestContext.java | 56 -
.../nifi/web/HttpServletRequestContext.java | 100 -
.../web/HttpServletRequestContextConfig.java | 119 -
.../nifi-web/nifi-jetty/.gitignore | 1 -
.../nifi-framework/nifi-web/nifi-jetty/pom.xml | 179 -
.../src/main/assembly/distribution.xml | 42 -
.../org/apache/nifi/web/server/JettyServer.java | 859 ----
.../server/ServerConfigurationException.java | 36 -
.../org/apache/nifi/web/webdefault.xml | 556 ---
.../nifi-web/nifi-ui-extension/pom.xml | 35 -
.../apache/nifi/ui/extension/UiExtension.java | 48 -
.../nifi/ui/extension/UiExtensionMapping.java | 51 -
.../nifi-web/nifi-web-api/.gitignore | 1 -
.../nifi-web/nifi-web-api/pom.xml | 325 --
.../org/apache/nifi/aop/MethodProfiler.java | 44 -
.../apache/nifi/audit/ControllerAuditor.java | 255 --
.../nifi/audit/ControllerServiceAuditor.java | 471 --
.../org/apache/nifi/audit/FunnelAuditor.java | 138 -
.../java/org/apache/nifi/audit/NiFiAuditor.java | 118 -
.../java/org/apache/nifi/audit/PortAuditor.java | 340 --
.../apache/nifi/audit/ProcessGroupAuditor.java | 259 --
.../org/apache/nifi/audit/ProcessorAuditor.java | 398 --
.../apache/nifi/audit/RelationshipAuditor.java | 406 --
.../nifi/audit/RemoteProcessGroupAuditor.java | 374 --
.../apache/nifi/audit/ReportingTaskAuditor.java | 348 --
.../org/apache/nifi/audit/SnippetAuditor.java | 510 ---
.../nifi/web/ClusterRequestException.java | 38 -
.../IllegalClusterResourceRequestException.java | 32 -
.../org/apache/nifi/web/NiFiCoreException.java | 32 -
.../org/apache/nifi/web/NiFiServiceFacade.java | 1444 ------
.../apache/nifi/web/NiFiServiceFacadeLock.java | 177 -
.../nifi/web/StandardNiFiContentAccess.java | 147 -
.../nifi/web/StandardNiFiServiceFacade.java | 3525 ---------------
.../StandardNiFiWebConfigurationContext.java | 731 ---
.../apache/nifi/web/StandardNiFiWebContext.java | 334 --
.../nifi/web/api/ApplicationResource.java | 397 --
.../nifi/web/api/BulletinBoardResource.java | 174 -
.../apache/nifi/web/api/ClusterResource.java | 1147 -----
.../apache/nifi/web/api/ConnectionResource.java | 905 ----
.../apache/nifi/web/api/ControllerResource.java | 1253 ------
.../nifi/web/api/ControllerServiceResource.java | 1072 -----
.../org/apache/nifi/web/api/FunnelResource.java | 624 ---
.../apache/nifi/web/api/HistoryResource.java | 528 ---
.../apache/nifi/web/api/InputPortResource.java | 648 ---
.../org/apache/nifi/web/api/LabelResource.java | 664 ---
.../org/apache/nifi/web/api/NodeResource.java | 431 --
.../apache/nifi/web/api/OutputPortResource.java | 649 ---
.../nifi/web/api/ProcessGroupResource.java | 1383 ------
.../apache/nifi/web/api/ProcessorResource.java | 920 ----
.../apache/nifi/web/api/ProvenanceResource.java | 1316 ------
.../web/api/RemoteProcessGroupResource.java | 1076 -----
.../nifi/web/api/ReportingTaskResource.java | 856 ----
.../apache/nifi/web/api/SnippetResource.java | 713 ---
.../nifi/web/api/SystemDiagnosticsResource.java | 111 -
.../apache/nifi/web/api/TemplateResource.java | 515 ---
.../apache/nifi/web/api/UserGroupResource.java | 465 --
.../org/apache/nifi/web/api/UserResource.java | 591 ---
.../api/config/AccessDeniedExceptionMapper.java | 54 -
.../config/AccountNotFoundExceptionMapper.java | 47 -
.../config/AdministrationExceptionMapper.java | 43 -
...ationCredentialsNotFoundExceptionMapper.java | 47 -
.../web/api/config/ClusterExceptionMapper.java | 42 -
...ectingNodeMutableRequestExceptionMapper.java | 49 -
...nectedNodeMutableRequestExceptionMapper.java | 49 -
.../config/IllegalArgumentExceptionMapper.java | 46 -
...alClusterResourceRequestExceptionMapper.java | 47 -
.../IllegalClusterStateExceptionMapper.java | 47 -
.../IllegalNodeDeletionExceptionMapper.java | 46 -
...IllegalNodeDisconnectionExceptionMapper.java | 46 -
.../IllegalNodeReconnectionExceptionMapper.java | 46 -
.../api/config/IllegalStateExceptionMapper.java | 46 -
.../IneligiblePrimaryNodeExceptionMapper.java | 47 -
.../config/InvalidRevisionExceptionMapper.java | 47 -
.../api/config/JsonMappingExceptionMapper.java | 46 -
.../api/config/JsonParseExceptionMapper.java | 46 -
.../config/MutableRequestExceptionMapper.java | 48 -
.../web/api/config/NiFiCoreExceptionMapper.java | 47 -
.../config/NoConnectedNodesExceptionMapper.java | 47 -
.../NoResponseFromNodesExceptionMapper.java | 48 -
.../NodeDisconnectionExceptionMapper.java | 47 -
.../config/NodeReconnectionExceptionMapper.java | 47 -
.../web/api/config/NotFoundExceptionMapper.java | 48 -
.../PrimaryRoleAssignmentExceptionMapper.java | 47 -
.../config/ResourceNotFoundExceptionMapper.java | 48 -
.../SafeModeMutableRequestExceptionMapper.java | 48 -
.../nifi/web/api/config/ThrowableMapper.java | 41 -
.../api/config/UnknownNodeExceptionMapper.java | 47 -
.../api/config/ValidationExceptionMapper.java | 49 -
.../config/WebApplicationExceptionMapper.java | 63 -
.../org/apache/nifi/web/api/dto/DtoFactory.java | 2430 ----------
.../request/BulletinBoardPatternParameter.java | 46 -
.../nifi/web/api/request/ClientIdParameter.java | 44 -
.../api/request/ConnectableTypeParameter.java | 41 -
.../nifi/web/api/request/DateTimeParameter.java | 62 -
.../nifi/web/api/request/DoubleParameter.java | 39 -
.../nifi/web/api/request/IntegerParameter.java | 39 -
.../nifi/web/api/request/LongParameter.java | 39 -
.../ApplicationStartupContextListener.java | 150 -
.../nifi/web/controller/ControllerFacade.java | 1425 ------
.../web/controller/StandardSearchContext.java | 64 -
.../org/apache/nifi/web/dao/ConnectionDAO.java | 109 -
.../nifi/web/dao/ControllerServiceDAO.java | 105 -
.../java/org/apache/nifi/web/dao/FunnelDAO.java | 83 -
.../java/org/apache/nifi/web/dao/LabelDAO.java | 74 -
.../java/org/apache/nifi/web/dao/PortDAO.java | 91 -
.../apache/nifi/web/dao/ProcessGroupDAO.java | 86 -
.../org/apache/nifi/web/dao/ProcessorDAO.java | 91 -
.../nifi/web/dao/RemoteProcessGroupDAO.java | 132 -
.../apache/nifi/web/dao/ReportingTaskDAO.java | 85 -
.../org/apache/nifi/web/dao/SnippetDAO.java | 88 -
.../org/apache/nifi/web/dao/TemplateDAO.java | 74 -
.../apache/nifi/web/dao/impl/ComponentDAO.java | 69 -
.../web/dao/impl/StandardConnectionDAO.java | 471 --
.../dao/impl/StandardControllerServiceDAO.java | 262 --
.../nifi/web/dao/impl/StandardFunnelDAO.java | 132 -
.../nifi/web/dao/impl/StandardInputPortDAO.java | 268 --
.../nifi/web/dao/impl/StandardLabelDAO.java | 136 -
.../web/dao/impl/StandardOutputPortDAO.java | 268 --
.../web/dao/impl/StandardProcessGroupDAO.java | 137 -
.../nifi/web/dao/impl/StandardProcessorDAO.java | 459 --
.../dao/impl/StandardRemoteProcessGroupDAO.java | 376 --
.../web/dao/impl/StandardReportingTaskDAO.java | 306 --
.../nifi/web/dao/impl/StandardSnippetDAO.java | 331 --
.../nifi/web/dao/impl/StandardTemplateDAO.java | 141 -
.../nifi/web/filter/NodeRequestFilter.java | 124 -
.../apache/nifi/web/filter/RequestLogger.java | 77 -
.../nifi/web/filter/ThreadLocalFilter.java | 53 -
.../org/apache/nifi/web/filter/TimerFilter.java | 69 -
.../ControllerServiceProviderFactoryBean.java | 68 -
.../OptimisticLockingManagerFactoryBean.java | 67 -
.../ReportingTaskProviderFactoryBean.java | 68 -
.../org/apache/nifi/web/util/Availability.java | 32 -
.../nifi/web/util/ClientResponseUtils.java | 48 -
.../org/apache/nifi/web/util/SnippetUtils.java | 609 ---
.../src/main/resources/images/bgNifiLogo.png | Bin 4232 -> 0 bytes
.../src/main/resources/images/nifi16.ico | Bin 1150 -> 0 bytes
.../src/main/resources/nifi-web-api-context.xml | 355 --
.../src/main/resources/templates/endpoint.hbs | 61 -
.../src/main/resources/templates/example.hbs | 18 -
.../src/main/resources/templates/index.html.hbs | 597 ---
.../src/main/resources/templates/operation.hbs | 104 -
.../src/main/resources/templates/type.hbs | 55 -
.../src/main/webapp/WEB-INF/web.xml | 100 -
.../apache/nifi/integration/NiFiWebApiTest.java | 247 -
.../accesscontrol/AdminAccessControlTest.java | 1010 -----
.../accesscontrol/DfmAccessControlTest.java | 1324 ------
.../ReadOnlyAccessControlTest.java | 982 ----
.../util/NiFiTestAuthorizationProvider.java | 179 -
.../nifi/integration/util/NiFiTestServer.java | 185 -
.../nifi/integration/util/NiFiTestUser.java | 261 --
.../integration/util/SourceTestProcessor.java | 49 -
.../util/TerminationTestProcessor.java | 44 -
.../nifi/integration/util/TestPrioritizer.java | 31 -
....apache.nifi.authorization.AuthorityProvider | 15 -
...org.apache.nifi.flowfile.FlowFilePrioritizer | 15 -
.../org.apache.nifi.processor.Processor | 16 -
.../access-control/authority-providers.xml | 24 -
.../access-control/controller-services.xml | 18 -
.../resources/access-control/localhost-ks.jks | Bin 2246 -> 0 bytes
.../resources/access-control/localhost-ts.jks | Bin 957 -> 0 bytes
.../resources/access-control/nifi.properties | 136 -
.../access-control/reporting-tasks.xml | 17 -
.../src/test/resources/logback-test.xml | 36 -
.../nifi-web/nifi-web-content-access/pom.xml | 25 -
.../java/org/apache/nifi/web/ContentAccess.java | 32 -
.../apache/nifi/web/ContentRequestContext.java | 51 -
.../apache/nifi/web/DownloadableContent.java | 62 -
.../nifi-web/nifi-web-content-viewer/.gitignore | 1 -
.../nifi-web/nifi-web-content-viewer/pom.xml | 105 -
.../nifi/web/ContentViewerController.java | 300 --
.../src/main/resources/META-INF/LICENSE | 232 -
.../src/main/resources/META-INF/NOTICE | 17 -
.../src/main/webapp/WEB-INF/jsp/footer.jsp | 20 -
.../src/main/webapp/WEB-INF/jsp/header.jsp | 96 -
.../src/main/webapp/WEB-INF/jsp/hexview.jsp | 32 -
.../src/main/webapp/WEB-INF/jsp/no-viewer.jsp | 20 -
.../src/main/webapp/WEB-INF/web.xml | 26 -
.../src/main/webapp/css/main.css | 113 -
.../main/webapp/js/hexview/hexview.default.css | 42 -
.../src/main/webapp/js/hexview/hexview.js | 232 -
.../nifi-web/nifi-web-docs/.gitignore | 1 -
.../nifi-web/nifi-web-docs/pom.xml | 62 -
.../nifi/web/docs/DocumentationController.java | 94 -
.../src/main/resources/META-INF/NOTICE | 19 -
.../main/webapp/WEB-INF/jsp/documentation.jsp | 130 -
.../WEB-INF/jsp/no-documentation-found.jsp | 31 -
.../src/main/webapp/WEB-INF/web.xml | 33 -
.../src/main/webapp/css/component-usage.css | 179 -
.../nifi-web-docs/src/main/webapp/css/main.css | 214 -
.../src/main/webapp/images/bgBannerFoot.png | Bin 189 -> 0 bytes
.../src/main/webapp/images/bgHeader.png | Bin 1697 -> 0 bytes
.../src/main/webapp/images/bgTableHeader.png | Bin 232 -> 0 bytes
.../src/main/webapp/js/application.js | 285 --
.../nifi-web/nifi-web-error/.gitignore | 1 -
.../nifi-web/nifi-web-error/pom.xml | 47 -
.../apache/nifi/web/filter/CatchAllFilter.java | 45 -
.../src/main/webapp/WEB-INF/web.xml | 26 -
.../nifi-web-error/src/main/webapp/index.jsp | 37 -
.../nifi-web-optimistic-locking/.gitignore | 1 -
.../nifi-web-optimistic-locking/pom.xml | 43 -
.../apache/nifi/web/ConfigurationRequest.java | 35 -
.../apache/nifi/web/ConfigurationResult.java | 41 -
.../apache/nifi/web/ConfigurationSnapshot.java | 94 -
.../org/apache/nifi/web/FlowModification.java | 57 -
.../nifi/web/OptimisticLockingManager.java | 53 -
.../web/StandardOptimisticLockingManager.java | 158 -
.../org/apache/nifi/web/UpdateRevision.java | 31 -
.../nifi-web/nifi-web-security/.gitignore | 1 -
.../nifi-web/nifi-web-security/pom.xml | 80 -
.../org/apache/nifi/web/security/DnUtils.java | 85 -
.../web/security/UntrustedProxyException.java | 34 -
.../anonymous/NiFiAnonymousUserFilter.java | 101 -
.../NiFiAuthenticationEntryPoint.java | 69 -
.../authorization/NiFiAuthorizationService.java | 175 -
.../authorization/NodeAuthorizedUserFilter.java | 128 -
.../nifi/web/security/user/NiFiUserDetails.java | 100 -
.../nifi/web/security/user/NiFiUserUtils.java | 91 -
.../x509/SubjectDnX509PrincipalExtractor.java | 32 -
.../security/x509/X509AuthenticationFilter.java | 317 --
.../security/x509/X509CertificateExtractor.java | 52 -
.../x509/ocsp/CertificateStatusException.java | 42 -
.../x509/ocsp/OcspCertificateValidator.java | 441 --
.../web/security/x509/ocsp/OcspRequest.java | 69 -
.../nifi/web/security/x509/ocsp/OcspStatus.java | 82 -
.../resources/nifi-web-security-context.xml | 83 -
.../NiFiAuthorizationServiceTest.java | 251 --
.../nifi-web/nifi-web-ui/.gitignore | 1 -
.../nifi-framework/nifi-web/nifi-web-ui/pom.xml | 656 ---
.../apache/nifi/web/filter/IeEdgeHeader.java | 56 -
.../apache/nifi/web/servlet/DownloadSvg.java | 89 -
.../src/main/resources/META-INF/LICENSE | 434 --
.../filters/bulletin-board-min.properties | 18 -
.../resources/filters/bulletin-board.properties | 25 -
.../resources/filters/canvas-min.properties | 18 -
.../main/resources/filters/canvas.properties | 67 -
.../resources/filters/cluster-min.properties | 18 -
.../main/resources/filters/cluster.properties | 26 -
.../resources/filters/counters-min.properties | 18 -
.../main/resources/filters/counters.properties | 26 -
.../resources/filters/history-min.properties | 18 -
.../main/resources/filters/history.properties | 27 -
.../resources/filters/provenance-min.properties | 18 -
.../resources/filters/provenance.properties | 26 -
.../resources/filters/summary-min.properties | 18 -
.../main/resources/filters/summary.properties | 32 -
.../resources/filters/templates-min.properties | 18 -
.../main/resources/filters/templates.properties | 26 -
.../main/resources/filters/users-min.properties | 18 -
.../src/main/resources/filters/users.properties | 26 -
.../webapp/WEB-INF/pages/bulletin-board.jsp | 50 -
.../src/main/webapp/WEB-INF/pages/canvas.jsp | 127 -
.../src/main/webapp/WEB-INF/pages/cluster.jsp | 62 -
.../src/main/webapp/WEB-INF/pages/counters.jsp | 61 -
.../src/main/webapp/WEB-INF/pages/history.jsp | 65 -
.../main/webapp/WEB-INF/pages/message-page.jsp | 34 -
.../main/webapp/WEB-INF/pages/provenance.jsp | 68 -
.../src/main/webapp/WEB-INF/pages/summary.jsp | 83 -
.../src/main/webapp/WEB-INF/pages/templates.jsp | 63 -
.../src/main/webapp/WEB-INF/pages/users.jsp | 71 -
.../webapp/WEB-INF/partials/banners-main.jsp | 19 -
.../webapp/WEB-INF/partials/banners-utility.jsp | 19 -
.../bulletin-board/bulletin-board-content.jsp | 40 -
.../WEB-INF/partials/canvas/about-dialog.jsp | 29 -
.../WEB-INF/partials/canvas/canvas-header.jsp | 56 -
.../canvas/connection-configuration.jsp | 180 -
.../partials/canvas/connections-dialog.jsp | 35 -
.../canvas/controller-service-configuration.jsp | 86 -
.../disable-controller-service-dialog.jsp | 76 -
.../canvas/enable-controller-service-dialog.jsp | 75 -
.../partials/canvas/fill-color-dialog.jsp | 46 -
.../WEB-INF/partials/canvas/flow-status.jsp | 55 -
.../canvas/instantiate-template-dialog.jsp | 22 -
.../partials/canvas/label-configuration.jsp | 38 -
.../WEB-INF/partials/canvas/navigation.jsp | 57 -
.../canvas/new-controller-service-dialog.jsp | 53 -
.../WEB-INF/partials/canvas/new-port-dialog.jsp | 27 -
.../canvas/new-process-group-dialog.jsp | 27 -
.../partials/canvas/new-processor-dialog.jsp | 48 -
.../canvas/new-remote-process-group-dialog.jsp | 27 -
.../canvas/new-reporting-task-dialog.jsp | 53 -
.../partials/canvas/new-template-dialog.jsp | 33 -
.../partials/canvas/port-configuration.jsp | 54 -
.../WEB-INF/partials/canvas/port-details.jsp | 39 -
.../canvas/process-group-configuration.jsp | 39 -
.../partials/canvas/process-group-details.jsp | 39 -
.../partials/canvas/processor-configuration.jsp | 206 -
.../WEB-INF/partials/canvas/registration.jsp | 44 -
.../canvas/remote-port-configuration.jsp | 42 -
.../remote-process-group-configuration.jsp | 54 -
.../canvas/remote-process-group-details.jsp | 54 -
.../canvas/remote-process-group-ports.jsp | 47 -
.../canvas/reporting-task-configuration.jsp | 107 -
.../canvas/secure-port-configuration.jsp | 82 -
.../partials/canvas/secure-port-details.jsp | 67 -
.../partials/canvas/settings-content.jsp | 94 -
.../webapp/WEB-INF/partials/canvas/shell.jsp | 27 -
.../partials/cluster/cluster-content.jsp | 39 -
.../partials/cluster/node-details-dialog.jsp | 42 -
.../WEB-INF/partials/connection-details.jsp | 123 -
.../partials/counters/counters-content.jsp | 39 -
.../partials/history/action-details-dialog.jsp | 27 -
.../partials/history/history-content.jsp | 36 -
.../partials/history/history-filter-dialog.jsp | 73 -
.../partials/history/history-purge-dialog.jsp | 42 -
.../webapp/WEB-INF/partials/message-pane.jsp | 23 -
.../main/webapp/WEB-INF/partials/ok-dialog.jsp | 22 -
.../WEB-INF/partials/processor-details.jsp | 154 -
.../provenance/lineage-query-dialog.jsp | 26 -
.../partials/provenance/provenance-content.jsp | 73 -
.../provenance-event-details-dialog.jsp | 196 -
.../provenance/provenance-query-dialog.jsp | 26 -
.../provenance/provenance-search-dialog.jsp | 101 -
.../WEB-INF/partials/status-history-dialog.jsp | 37 -
.../cluster-connection-summary-dialog.jsp | 36 -
.../cluster-input-port-summary-dialog.jsp | 36 -
.../cluster-output-port-summary-dialog.jsp | 36 -
.../cluster-process-group-summary-dialog.jsp | 36 -
.../cluster-processor-summary-dialog.jsp | 36 -
...ster-remote-process-group-summary-dialog.jsp | 36 -
.../partials/summary/summary-content.jsp | 67 -
.../summary/system-diagnostics-dialog.jsp | 131 -
.../summary/view-single-node-dialog.jsp | 24 -
.../partials/templates/templates-content.jsp | 59 -
.../partials/users/group-revoke-dialog.jsp | 22 -
.../partials/users/group-roles-dialog.jsp | 52 -
.../partials/users/user-delete-dialog.jsp | 23 -
.../partials/users/user-details-dialog.jsp | 56 -
.../partials/users/user-group-dialog.jsp | 27 -
.../partials/users/user-revoke-dialog.jsp | 23 -
.../partials/users/user-roles-dialog.jsp | 60 -
.../WEB-INF/partials/users/users-content.jsp | 46 -
.../webapp/WEB-INF/partials/yes-no-dialog.jsp | 22 -
.../nifi-web-ui/src/main/webapp/WEB-INF/web.xml | 153 -
.../nifi-web-ui/src/main/webapp/css/about.css | 53 -
.../nifi-web-ui/src/main/webapp/css/banner.css | 80 -
.../src/main/webapp/css/bulletin-board.css | 238 -
.../nifi-web-ui/src/main/webapp/css/canvas.css | 45 -
.../nifi-web-ui/src/main/webapp/css/cluster.css | 152 -
.../webapp/css/connection-configuration.css | 167 -
.../src/main/webapp/css/connection-details.css | 76 -
.../src/main/webapp/css/controller-service.css | 304 --
.../src/main/webapp/css/counters.css | 134 -
.../nifi-web-ui/src/main/webapp/css/dialog.css | 332 --
.../src/main/webapp/css/flow-status.css | 110 -
.../nifi-web-ui/src/main/webapp/css/graph.css | 262 --
.../nifi-web-ui/src/main/webapp/css/header.css | 683 ---
.../nifi-web-ui/src/main/webapp/css/history.css | 216 -
.../src/main/webapp/css/label-configuration.css | 54 -
.../nifi-web-ui/src/main/webapp/css/main.css | 314 --
.../src/main/webapp/css/message-page.css | 30 -
.../src/main/webapp/css/message-pane.css | 44 -
.../src/main/webapp/css/navigation.css | 292 --
.../css/new-controller-service-dialog.css | 152 -
.../main/webapp/css/new-processor-dialog.css | 131 -
.../webapp/css/new-reporting-task-dialog.css | 152 -
.../src/main/webapp/css/port-configuration.css | 190 -
.../src/main/webapp/css/port-details.css | 53 -
.../webapp/css/process-group-configuration.css | 30 -
.../main/webapp/css/process-group-details.css | 22 -
.../main/webapp/css/processor-configuration.css | 235 -
.../src/main/webapp/css/processor-details.css | 92 -
.../src/main/webapp/css/provenance.css | 660 ---
.../src/main/webapp/css/registration.css | 45 -
.../css/remote-process-group-configuration.css | 230 -
.../src/main/webapp/css/reporting-task.css | 116 -
.../nifi-web-ui/src/main/webapp/css/reset.css | 43 -
.../src/main/webapp/css/settings.css | 175 -
.../nifi-web-ui/src/main/webapp/css/shell.css | 98 -
.../src/main/webapp/css/status-history.css | 197 -
.../nifi-web-ui/src/main/webapp/css/summary.css | 851 ----
.../src/main/webapp/css/templates.css | 229 -
.../nifi-web-ui/src/main/webapp/css/users.css | 254 --
.../src/main/webapp/images/addConnect.png | Bin 1996 -> 0 bytes
.../src/main/webapp/images/bg-error.png | Bin 63766 -> 0 bytes
.../src/main/webapp/images/bgBannerFoot.png | Bin 189 -> 0 bytes
.../src/main/webapp/images/bgBannerHead.png | Bin 422 -> 0 bytes
.../src/main/webapp/images/bgBreadcrumbs.png | Bin 2130 -> 0 bytes
.../src/main/webapp/images/bgButton.png | Bin 234 -> 0 bytes
.../src/main/webapp/images/bgButtonOver.png | Bin 234 -> 0 bytes
.../src/main/webapp/images/bgButtonSelected.png | Bin 166 -> 0 bytes
.../main/webapp/images/bgButtonSelectedOver.png | Bin 168 -> 0 bytes
.../src/main/webapp/images/bgContextMenu.png | Bin 746 -> 0 bytes
.../src/main/webapp/images/bgControlsInset.png | Bin 618 -> 0 bytes
.../src/main/webapp/images/bgHeader.png | Bin 1697 -> 0 bytes
.../src/main/webapp/images/bgInputText.png | Bin 139 -> 0 bytes
.../src/main/webapp/images/bgNifiLogo.png | Bin 4232 -> 0 bytes
.../src/main/webapp/images/bgNifiToolbar.png | Bin 2123 -> 0 bytes
.../main/webapp/images/bgNifiToolbarEdge.png | Bin 541 -> 0 bytes
.../src/main/webapp/images/bgOutline.png | Bin 585 -> 0 bytes
.../src/main/webapp/images/bgPanCenter.png | Bin 362 -> 0 bytes
.../src/main/webapp/images/bgPanZoom.png | Bin 832 -> 0 bytes
.../webapp/images/bgProcessGroupDetailsArea.png | Bin 6596 -> 0 bytes
.../main/webapp/images/bgProcessorStatArea.png | Bin 4062 -> 0 bytes
.../images/bgRemoteProcessGroupDetailsArea.png | Bin 4433 -> 0 bytes
.../src/main/webapp/images/bgSearchInput.png | Bin 221 -> 0 bytes
.../src/main/webapp/images/bgShellClose.png | Bin 169 -> 0 bytes
.../src/main/webapp/images/bgStatusPanel.png | Bin 229 -> 0 bytes
.../src/main/webapp/images/bgTabContainer.png | Bin 234 -> 0 bytes
.../src/main/webapp/images/bgTableHeader.png | Bin 232 -> 0 bytes
.../main/webapp/images/bgToolbarBtnBorder.png | Bin 227 -> 0 bytes
.../src/main/webapp/images/blueBtnBg-over.jpg | Bin 356 -> 0 bytes
.../src/main/webapp/images/blueBtnBg.jpg | Bin 356 -> 0 bytes
.../src/main/webapp/images/buttonColor.png | Bin 1679 -> 0 bytes
.../src/main/webapp/images/buttonCopy.png | Bin 1166 -> 0 bytes
.../src/main/webapp/images/buttonDelete.png | Bin 1837 -> 0 bytes
.../src/main/webapp/images/buttonDisable.png | Bin 1737 -> 0 bytes
.../src/main/webapp/images/buttonEnable.png | Bin 1375 -> 0 bytes
.../src/main/webapp/images/buttonGroup.png | Bin 880 -> 0 bytes
.../src/main/webapp/images/buttonOutline.png | Bin 495 -> 0 bytes
.../src/main/webapp/images/buttonPanDown.png | Bin 554 -> 0 bytes
.../src/main/webapp/images/buttonPanLeft.png | Bin 588 -> 0 bytes
.../src/main/webapp/images/buttonPanRight.png | Bin 530 -> 0 bytes
.../src/main/webapp/images/buttonPanUp.png | Bin 551 -> 0 bytes
.../src/main/webapp/images/buttonPaste.png | Bin 1372 -> 0 bytes
.../src/main/webapp/images/buttonRefresh.png | Bin 915 -> 0 bytes
.../src/main/webapp/images/buttonRun.png | Bin 1163 -> 0 bytes
.../src/main/webapp/images/buttonStop.png | Bin 869 -> 0 bytes
.../src/main/webapp/images/buttonTemplate.png | Bin 1290 -> 0 bytes
.../src/main/webapp/images/buttonZoom100.png | Bin 449 -> 0 bytes
.../src/main/webapp/images/buttonZoomFit.png | Bin 499 -> 0 bytes
.../src/main/webapp/images/buttonZoomIn.png | Bin 435 -> 0 bytes
.../src/main/webapp/images/buttonZoomOut.png | Bin 339 -> 0 bytes
.../src/main/webapp/images/cluster.png | Bin 326 -> 0 bytes
.../src/main/webapp/images/grayBtnBg-over.jpg | Bin 319 -> 0 bytes
.../src/main/webapp/images/grayBtnBg.jpg | Bin 317 -> 0 bytes
.../nifi-web-ui/src/main/webapp/images/grid.gif | Bin 135 -> 0 bytes
.../src/main/webapp/images/headerTabBg.gif | Bin 131 -> 0 bytes
.../src/main/webapp/images/iconAdminUser.png | Bin 1960 -> 0 bytes
.../src/main/webapp/images/iconAlert.png | Bin 1396 -> 0 bytes
.../src/main/webapp/images/iconAlertDialog.png | Bin 1241 -> 0 bytes
.../src/main/webapp/images/iconAutoRefresh.png | Bin 3102 -> 0 bytes
.../src/main/webapp/images/iconBulletin.png | Bin 1066 -> 0 bytes
.../main/webapp/images/iconBulletinBoard.png | Bin 2523 -> 0 bytes
.../src/main/webapp/images/iconCenterView.png | Bin 338 -> 0 bytes
.../src/main/webapp/images/iconChart.png | Bin 510 -> 0 bytes
.../src/main/webapp/images/iconClose.png | Bin 231 -> 0 bytes
.../src/main/webapp/images/iconCluster.png | Bin 2112 -> 0 bytes
.../src/main/webapp/images/iconClusterSmall.png | Bin 757 -> 0 bytes
.../main/webapp/images/iconCollapseChildren.png | Bin 571 -> 0 bytes
.../main/webapp/images/iconCollapseParents.png | Bin 645 -> 0 bytes
.../src/main/webapp/images/iconColor.png | Bin 738 -> 0 bytes
.../src/main/webapp/images/iconCommit.png | Bin 445 -> 0 bytes
.../src/main/webapp/images/iconConfigure.png | Bin 696 -> 0 bytes
.../src/main/webapp/images/iconConnect.png | Bin 589 -> 0 bytes
.../src/main/webapp/images/iconConnection.png | Bin 1517 -> 0 bytes
.../src/main/webapp/images/iconCopy.png | Bin 514 -> 0 bytes
.../src/main/webapp/images/iconCounter.png | Bin 1369 -> 0 bytes
.../src/main/webapp/images/iconDelete.png | Bin 670 -> 0 bytes
.../src/main/webapp/images/iconDetails.png | Bin 549 -> 0 bytes
.../src/main/webapp/images/iconDisable.png | Bin 764 -> 0 bytes
.../src/main/webapp/images/iconDisconnect.png | Bin 569 -> 0 bytes
.../src/main/webapp/images/iconEdit.png | Bin 493 -> 0 bytes
.../src/main/webapp/images/iconEditButton.png | Bin 915 -> 0 bytes
.../src/main/webapp/images/iconEnable.png | Bin 472 -> 0 bytes
.../main/webapp/images/iconExpandChildren.png | Bin 521 -> 0 bytes
.../main/webapp/images/iconExpandParents.png | Bin 585 -> 0 bytes
.../src/main/webapp/images/iconExport.png | Bin 453 -> 0 bytes
.../main/webapp/images/iconExportLineage.png | Bin 405 -> 0 bytes
.../src/main/webapp/images/iconFlowSettings.png | Bin 2624 -> 0 bytes
.../src/main/webapp/images/iconFunnel.png | Bin 1223 -> 0 bytes
.../src/main/webapp/images/iconGoTo.png | Bin 448 -> 0 bytes
.../src/main/webapp/images/iconHistory.png | Bin 2347 -> 0 bytes
.../src/main/webapp/images/iconInfo.png | Bin 550 -> 0 bytes
.../src/main/webapp/images/iconInputPort.png | Bin 1842 -> 0 bytes
.../main/webapp/images/iconInputPortSmall.png | Bin 532 -> 0 bytes
.../webapp/images/iconIsolatedProcessor.png | Bin 1781 -> 0 bytes
.../src/main/webapp/images/iconLineage.png | Bin 432 -> 0 bytes
.../src/main/webapp/images/iconLoading.gif | Bin 2438 -> 0 bytes
.../src/main/webapp/images/iconMoveToParent.png | Bin 215 -> 0 bytes
.../src/main/webapp/images/iconNotSecure.png | Bin 137 -> 0 bytes
.../src/main/webapp/images/iconOutputPort.png | Bin 1658 -> 0 bytes
.../main/webapp/images/iconOutputPortSmall.png | Bin 459 -> 0 bytes
.../src/main/webapp/images/iconPaste.png | Bin 601 -> 0 bytes
.../src/main/webapp/images/iconPopOut.png | Bin 217 -> 0 bytes
.../webapp/images/iconPortNotTransmitting.png | Bin 231 -> 0 bytes
.../src/main/webapp/images/iconPortRemoved.png | Bin 456 -> 0 bytes
.../main/webapp/images/iconPortTransmitting.png | Bin 235 -> 0 bytes
.../src/main/webapp/images/iconPrimary.png | Bin 647 -> 0 bytes
.../src/main/webapp/images/iconProcessGroup.png | Bin 1314 -> 0 bytes
.../src/main/webapp/images/iconProcessor.png | Bin 1446 -> 0 bytes
.../src/main/webapp/images/iconProvenance.png | Bin 1104 -> 0 bytes
.../src/main/webapp/images/iconRefresh.png | Bin 492 -> 0 bytes
.../src/main/webapp/images/iconRemotePorts.png | Bin 456 -> 0 bytes
.../webapp/images/iconRemoteProcessGroup.png | Bin 674 -> 0 bytes
.../src/main/webapp/images/iconReporting.png | Bin 803 -> 0 bytes
.../src/main/webapp/images/iconResetCounter.png | Bin 304 -> 0 bytes
.../src/main/webapp/images/iconResize.png | Bin 142 -> 0 bytes
.../src/main/webapp/images/iconRevoke.png | Bin 676 -> 0 bytes
.../src/main/webapp/images/iconRun.png | Bin 538 -> 0 bytes
.../src/main/webapp/images/iconSecure.png | Bin 133 -> 0 bytes
.../src/main/webapp/images/iconSmallFunnel.png | Bin 455 -> 0 bytes
.../main/webapp/images/iconSmallInputPort.png | Bin 216 -> 0 bytes
.../main/webapp/images/iconSmallOutputPort.png | Bin 264 -> 0 bytes
.../webapp/images/iconSmallProcessGroup.png | Bin 479 -> 0 bytes
.../main/webapp/images/iconSmallProcessor.png | Bin 647 -> 0 bytes
.../webapp/images/iconSmallRelationship.png | Bin 770 -> 0 bytes
.../images/iconSmallRemoteProcessGroup.png | Bin 402 -> 0 bytes
.../src/main/webapp/images/iconStop.png | Bin 402 -> 0 bytes
.../src/main/webapp/images/iconTemplates.png | Bin 2659 -> 0 bytes
.../src/main/webapp/images/iconToFront.png | Bin 475 -> 0 bytes
.../webapp/images/iconTransmissionActive.png | Bin 1330 -> 0 bytes
.../webapp/images/iconTransmissionInactive.png | Bin 1248 -> 0 bytes
.../src/main/webapp/images/iconTwistArrow.png | Bin 256 -> 0 bytes
.../src/main/webapp/images/iconUsage.png | Bin 470 -> 0 bytes
.../src/main/webapp/images/inputCheckbox.png | Bin 389 -> 0 bytes
.../src/main/webapp/images/loadAnimation.gif | Bin 10789 -> 0 bytes
.../src/main/webapp/images/nifi16.ico | Bin 1150 -> 0 bytes
.../src/main/webapp/images/nifi_about.png | Bin 16323 -> 0 bytes
.../src/main/webapp/images/panelBg.jpg | Bin 312 -> 0 bytes
.../src/main/webapp/images/portRemoved.png | Bin 1090 -> 0 bytes
.../src/main/webapp/images/portRunning.png | Bin 337 -> 0 bytes
.../src/main/webapp/images/portStopped.png | Bin 192 -> 0 bytes
.../src/main/webapp/images/spacer.png | Bin 110 -> 0 bytes
.../src/main/webapp/images/starburst.png | Bin 190 -> 0 bytes
.../src/main/webapp/images/tabBg.jpg | Bin 320 -> 0 bytes
.../src/main/webapp/images/toolbarBg.gif | Bin 45 -> 0 bytes
.../main/webapp/images/toolboxIconFunnel.png | Bin 2409 -> 0 bytes
.../main/webapp/images/toolboxIconInputPort.png | Bin 2680 -> 0 bytes
.../src/main/webapp/images/toolboxIconLabel.png | Bin 2871 -> 0 bytes
.../webapp/images/toolboxIconOutputPort.png | Bin 2209 -> 0 bytes
.../webapp/images/toolboxIconProcessGroup.png | Bin 4176 -> 0 bytes
.../main/webapp/images/toolboxIconProcessor.png | Bin 4426 -> 0 bytes
.../images/toolboxIconRemoteProcessGroup.png | Bin 5653 -> 0 bytes
.../main/webapp/images/toolboxIconTemplate.png | Bin 2913 -> 0 bytes
.../images/transmissionSwitchDisabled.png | Bin 1240 -> 0 bytes
.../webapp/images/transmissionSwitchEnabled.png | Bin 1600 -> 0 bytes
.../src/main/webapp/images/ungroup.png | Bin 3409 -> 0 bytes
.../src/main/webapp/js/codemirror/LICENSE | 19 -
.../js/codemirror/addon/fold/foldgutter.css | 20 -
.../js/codemirror/addon/hint/show-hint.css | 38 -
.../js/codemirror/lib/codemirror-compressed.js | 26 -
.../webapp/js/codemirror/lib/codemirror.css | 318 --
.../nifi-web-ui/src/main/webapp/js/d3/LICENSE | 26 -
.../nifi-web-ui/src/main/webapp/js/d3/d3.min.js | 5 -
.../webapp/js/jquery/combo/jquery.combo.css | 96 -
.../main/webapp/js/jquery/combo/jquery.combo.js | 311 --
.../main/webapp/js/jquery/combo/menuArrow.png | Bin 251 -> 0 bytes
.../webapp/js/jquery/combo/menuArrowSmall.png | Bin 233 -> 0 bytes
.../main/webapp/js/jquery/jquery-2.1.1.min.js | 4 -
.../src/main/webapp/js/jquery/jquery.center.js | 35 -
.../src/main/webapp/js/jquery/jquery.count.js | 99 -
.../src/main/webapp/js/jquery/jquery.each.js | 31 -
.../main/webapp/js/jquery/jquery.ellipsis.js | 164 -
.../js/jquery/jquery.event.drag-2.2.min.js | 6 -
.../main/webapp/js/jquery/jquery.form.min.js | 11 -
.../src/main/webapp/js/jquery/jquery.tab.js | 69 -
.../js/jquery/minicolors/jquery.minicolors.css | 259 --
.../jquery/minicolors/jquery.minicolors.min.js | 9 -
.../js/jquery/minicolors/jquery.minicolors.png | Bin 77459 -> 0 bytes
.../webapp/js/jquery/modal/jquery.modal.css | 112 -
.../main/webapp/js/jquery/modal/jquery.modal.js | 249 --
.../js/jquery/nfeditor/jquery.nfeditor.css | 73 -
.../js/jquery/nfeditor/jquery.nfeditor.js | 318 --
.../js/jquery/nfeditor/languages/nfel.css | 51 -
.../webapp/js/jquery/nfeditor/languages/nfel.js | 834 ----
.../jquery/propertytable/buttonNewProperty.png | Bin 590 -> 0 bytes
.../propertytable/jquery.propertytable.css | 216 -
.../propertytable/jquery.propertytable.js | 1647 -------
.../webapp/js/jquery/qtip2/jquery.qtip.min.css | 3 -
.../webapp/js/jquery/qtip2/jquery.qtip.min.js | 5 -
.../webapp/js/jquery/slickgrid/MIT-LICENSE.txt | 20 -
.../js/jquery/slickgrid/css/images/collapse.gif | Bin 846 -> 0 bytes
.../js/jquery/slickgrid/css/images/expand.gif | Bin 851 -> 0 bytes
.../js/jquery/slickgrid/css/images/sort-asc.gif | Bin 59 -> 0 bytes
.../jquery/slickgrid/css/images/sort-desc.gif | Bin 59 -> 0 bytes
.../slickgrid/css/slick-default-theme.css | 135 -
.../js/jquery/slickgrid/css/slick.grid.css | 158 -
.../slickgrid/plugins/slick.autotooltips.js | 83 -
.../plugins/slick.cellrangedecorator.js | 66 -
.../plugins/slick.cellrangeselector.js | 113 -
.../plugins/slick.cellselectionmodel.js | 154 -
.../plugins/slick.rowselectionmodel.js | 187 -
.../webapp/js/jquery/slickgrid/slick.core.js | 467 --
.../js/jquery/slickgrid/slick.dataview.js | 1126 -----
.../webapp/js/jquery/slickgrid/slick.editors.js | 512 ---
.../js/jquery/slickgrid/slick.formatters.js | 59 -
.../webapp/js/jquery/slickgrid/slick.grid.js | 3422 --------------
.../webapp/js/jquery/tabbs/jquery.tabbs.css | 48 -
.../main/webapp/js/jquery/tabbs/jquery.tabbs.js | 98 -
.../js/jquery/tagcloud/jquery.tagcloud.css | 62 -
.../js/jquery/tagcloud/jquery.tagcloud.js | 226 -
.../ui-smoothness/images/animated-overlay.gif | Bin 1738 -> 0 bytes
.../images/ui-bg_flat_0_aaaaaa_40x100.png | Bin 212 -> 0 bytes
.../images/ui-bg_flat_75_ffffff_40x100.png | Bin 208 -> 0 bytes
.../images/ui-bg_glass_55_fbf9ee_1x400.png | Bin 335 -> 0 bytes
.../images/ui-bg_glass_65_ffffff_1x400.png | Bin 207 -> 0 bytes
.../images/ui-bg_glass_75_dadada_1x400.png | Bin 262 -> 0 bytes
.../images/ui-bg_glass_75_e6e6e6_1x400.png | Bin 262 -> 0 bytes
.../images/ui-bg_glass_95_fef1ec_1x400.png | Bin 332 -> 0 bytes
.../ui-bg_highlight-soft_75_cccccc_1x100.png | Bin 280 -> 0 bytes
.../images/ui-icons_222222_256x240.png | Bin 6922 -> 0 bytes
.../images/ui-icons_2e83ff_256x240.png | Bin 4549 -> 0 bytes
.../images/ui-icons_454545_256x240.png | Bin 6992 -> 0 bytes
.../images/ui-icons_888888_256x240.png | Bin 6999 -> 0 bytes
.../images/ui-icons_cd0a0a_256x240.png | Bin 4549 -> 0 bytes
.../ui-smoothness/jquery-ui-1.10.4.min.css | 7 -
.../ui-smoothness/jquery-ui-1.10.4.min.js | 7 -
.../nifi-web-ui/src/main/webapp/js/json2.js | 475 --
.../js/nf/bulletin-board/nf-bulletin-board.js | 425 --
.../src/main/webapp/js/nf/canvas/nf-actions.js | 1184 -----
.../src/main/webapp/js/nf/canvas/nf-birdseye.js | 343 --
.../webapp/js/nf/canvas/nf-canvas-header.js | 356 --
.../webapp/js/nf/canvas/nf-canvas-toolbar.js | 165 -
.../webapp/js/nf/canvas/nf-canvas-toolbox.js | 1133 -----
.../main/webapp/js/nf/canvas/nf-canvas-utils.js | 1379 ------
.../src/main/webapp/js/nf/canvas/nf-canvas.js | 1641 -------
.../main/webapp/js/nf/canvas/nf-clipboard.js | 99 -
.../main/webapp/js/nf/canvas/nf-connectable.js | 250 --
.../js/nf/canvas/nf-connection-configuration.js | 1384 ------
.../main/webapp/js/nf/canvas/nf-connection.js | 1536 -------
.../main/webapp/js/nf/canvas/nf-context-menu.js | 460 --
.../js/nf/canvas/nf-controller-service.js | 1906 --------
.../main/webapp/js/nf/canvas/nf-custom-ui.js | 47 -
.../main/webapp/js/nf/canvas/nf-draggable.js | 322 --
.../src/main/webapp/js/nf/canvas/nf-funnel.js | 282 --
.../src/main/webapp/js/nf/canvas/nf-go-to.js | 733 ---
.../webapp/js/nf/canvas/nf-graph-control.js | 140 -
.../src/main/webapp/js/nf/canvas/nf-graph.js | 201 -
.../js/nf/canvas/nf-label-configuration.js | 135 -
.../src/main/webapp/js/nf/canvas/nf-label.js | 562 ---
.../js/nf/canvas/nf-port-configuration.js | 173 -
.../main/webapp/js/nf/canvas/nf-port-details.js | 66 -
.../src/main/webapp/js/nf/canvas/nf-port.js | 635 ---
.../nf/canvas/nf-process-group-configuration.js | 110 -
.../js/nf/canvas/nf-process-group-details.js | 66 -
.../webapp/js/nf/canvas/nf-process-group.js | 1067 -----
.../js/nf/canvas/nf-processor-configuration.js | 782 ----
.../main/webapp/js/nf/canvas/nf-processor.js | 834 ----
.../main/webapp/js/nf/canvas/nf-registration.js | 71 -
.../nf-remote-process-group-configuration.js | 130 -
.../canvas/nf-remote-process-group-details.js | 72 -
.../nf/canvas/nf-remote-process-group-ports.js | 528 ---
.../js/nf/canvas/nf-remote-process-group.js | 1063 -----
.../webapp/js/nf/canvas/nf-reporting-task.js | 735 ---
.../src/main/webapp/js/nf/canvas/nf-search.js | 179 -
.../nf/canvas/nf-secure-port-configuration.js | 384 --
.../js/nf/canvas/nf-secure-port-details.js | 121 -
.../main/webapp/js/nf/canvas/nf-selectable.js | 61 -
.../src/main/webapp/js/nf/canvas/nf-settings.js | 1752 --------
.../src/main/webapp/js/nf/canvas/nf-snippet.js | 215 -
.../src/main/webapp/js/nf/canvas/nf-storage.js | 139 -
.../webapp/js/nf/canvas/nf-toolbar-action.js | 96 -
.../webapp/js/nf/cluster/nf-cluster-table.js | 620 ---
.../src/main/webapp/js/nf/cluster/nf-cluster.js | 157 -
.../webapp/js/nf/counters/nf-counters-table.js | 306 --
.../main/webapp/js/nf/counters/nf-counters.js | 156 -
.../webapp/js/nf/history/nf-history-model.js | 221 -
.../webapp/js/nf/history/nf-history-table.js | 446 --
.../src/main/webapp/js/nf/history/nf-history.js | 157 -
.../src/main/webapp/js/nf/nf-client.js | 59 -
.../src/main/webapp/js/nf/nf-common.js | 997 -----
.../main/webapp/js/nf/nf-connection-details.js | 470 --
.../src/main/webapp/js/nf/nf-dialog.js | 130 -
.../src/main/webapp/js/nf/nf-namespace.js | 20 -
.../main/webapp/js/nf/nf-processor-details.js | 270 --
.../src/main/webapp/js/nf/nf-shell.js | 170 -
.../src/main/webapp/js/nf/nf-status-history.js | 1323 ------
.../js/nf/provenance/nf-provenance-lineage.js | 1432 ------
.../js/nf/provenance/nf-provenance-table.js | 1367 ------
.../webapp/js/nf/provenance/nf-provenance.js | 207 -
.../webapp/js/nf/summary/nf-cluster-search.js | 183 -
.../webapp/js/nf/summary/nf-summary-table.js | 2655 -----------
.../src/main/webapp/js/nf/summary/nf-summary.js | 171 -
.../js/nf/templates/nf-templates-table.js | 335 --
.../main/webapp/js/nf/templates/nf-templates.js | 238 -
.../main/webapp/js/nf/users/nf-users-table.js | 1075 -----
.../src/main/webapp/js/nf/users/nf-users.js | 149 -
.../nifi-framework/nifi-web/pom.xml | 71 -
.../nifi-framework/pom.xml | 50 -
.../nifi-framework-bundle/pom.xml | 123 -
.../nifi-geo-bundle/nifi-geo-nar/pom.xml | 33 -
.../src/main/resources/META-INF/NOTICE | 68 -
.../nifi-geo-processors/.gitignore | 1 -
.../nifi-geo-bundle/nifi-geo-processors/pom.xml | 43 -
.../org/apache/nifi/processors/GeoEnrichIP.java | 217 -
.../nifi/processors/maxmind/DatabaseReader.java | 274 --
.../org.apache.nifi.processor.Processor | 16 -
nifi/nifi-nar-bundles/nifi-geo-bundle/pom.xml | 42 -
.../nifi-hadoop-bundle/nifi-hadoop-nar/pom.xml | 35 -
.../src/main/resources/META-INF/NOTICE | 35 -
.../nifi-hdfs-processors/pom.xml | 65 -
.../hadoop/AbstractHadoopProcessor.java | 297 --
.../hadoop/CreateHadoopSequenceFile.java | 168 -
.../nifi/processors/hadoop/FetchHDFS.java | 125 -
...lowFileStreamUnpackerSequenceFileWriter.java | 170 -
.../apache/nifi/processors/hadoop/GetHDFS.java | 543 ---
.../processors/hadoop/GetHDFSSequenceFile.java | 139 -
.../nifi/processors/hadoop/KeyValueReader.java | 129 -
.../apache/nifi/processors/hadoop/ListHDFS.java | 481 --
.../apache/nifi/processors/hadoop/PutHDFS.java | 409 --
.../hadoop/SequenceFileWriterImpl.java | 119 -
.../hadoop/TarUnpackerSequenceFileWriter.java | 54 -
.../nifi/processors/hadoop/ValueReader.java | 115 -
.../hadoop/ZipUnpackerSequenceFileWriter.java | 57 -
.../hadoop/util/ByteFilteringOutputStream.java | 155 -
.../processors/hadoop/util/HDFSListing.java | 83 -
.../hadoop/util/InputStreamWritable.java | 59 -
.../nifi/processors/hadoop/util/LongSerDe.java | 48 -
.../hadoop/util/OutputStreamWritable.java | 83 -
.../hadoop/util/SequenceFileReader.java | 28 -
.../hadoop/util/SequenceFileWriter.java | 38 -
.../processors/hadoop/util/StringSerDe.java | 44 -
.../org.apache.nifi.processor.Processor | 20 -
.../additionalDetails.html | 46 -
.../processors/hadoop/AbstractHadoopTest.java | 90 -
.../nifi/processors/hadoop/GetHDFSTest.java | 155 -
.../nifi/processors/hadoop/PutHDFSTest.java | 173 -
.../hadoop/SimpleHadoopProcessor.java | 30 -
.../hadoop/TestCreateHadoopSequenceFile.java | 176 -
.../nifi/processors/hadoop/TestListHDFS.java | 357 --
.../src/test/resources/core-site-broken.xml | 25 -
.../src/test/resources/core-site.xml | 25 -
.../resources/testdata/13545312236534130.tar | Bin 1505280 -> 0 bytes
.../resources/testdata/13545423550275052.zip | Bin 1500841 -> 0 bytes
.../resources/testdata/13545479542069498.pkg | Bin 1500429 -> 0 bytes
.../src/test/resources/testdata/randombytes-1 | Bin 500000 -> 0 bytes
.../test/resources/testdata/randombytes-1.gz | Bin 500175 -> 0 bytes
.../src/test/resources/testdata/randombytes-2 | Bin 500000 -> 0 bytes
.../src/test/resources/testdata/randombytes-3 | Bin 500000 -> 0 bytes
.../nifi-nar-bundles/nifi-hadoop-bundle/pom.xml | 38 -
.../nifi-hadoop-libraries-nar/pom.xml | 55 -
.../src/main/resources/META-INF/LICENSE | 387 --
.../src/main/resources/META-INF/NOTICE | 300 --
.../nifi-hadoop-libraries-bundle/pom.xml | 29 -
.../nifi-hl7-bundle/nifi-hl7-nar/pom.xml | 36 -
.../src/main/resources/META-INF/LICENSE | 238 -
.../src/main/resources/META-INF/NOTICE | 29 -
.../nifi-hl7-processors/.gitignore | 1 -
.../nifi-hl7-bundle/nifi-hl7-processors/pom.xml | 120 -
.../processors/hl7/ExtractHL7Attributes.java | 244 -
.../apache/nifi/processors/hl7/RouteHL7.java | 221 -
.../org.apache.nifi.processor.Processor | 16 -
.../hl7/TestExtractHL7Attributes.java | 77 -
.../src/test/resources/hypoglycemia.hl7 | 5 -
nifi/nifi-nar-bundles/nifi-hl7-bundle/pom.xml | 33 -
nifi/nifi-nar-bundles/nifi-jetty-bundle/pom.xml | 77 -
.../src/main/resources/META-INF/NOTICE | 37 -
.../nifi-kafka-bundle/nifi-kafka-nar/pom.xml | 31 -
.../src/main/resources/META-INF/LICENSE | 299 --
.../src/main/resources/META-INF/NOTICE | 72 -
.../nifi-kafka-processors/pom.xml | 69 -
.../apache/nifi/processors/kafka/GetKafka.java | 356 --
.../apache/nifi/processors/kafka/PutKafka.java | 561 ---
.../org.apache.nifi.processor.Processor | 16 -
.../additionalDetails.html | 45 -
.../additionalDetails.html | 45 -
.../nifi/processors/kafka/TestGetKafka.java | 160 -
.../nifi/processors/kafka/TestPutKafka.java | 440 --
nifi/nifi-nar-bundles/nifi-kafka-bundle/pom.xml | 37 -
.../nifi-kite-bundle/nifi-kite-nar/pom.xml | 85 -
.../src/main/resources/META-INF/NOTICE | 51 -
.../nifi-kite-processors/pom.xml | 133 -
.../processors/kite/AbstractKiteProcessor.java | 220 -
.../processors/kite/AvroRecordConverter.java | 320 --
.../apache/nifi/processors/kite/AvroUtil.java | 38 -
.../nifi/processors/kite/ConvertAvroSchema.java | 339 --
.../nifi/processors/kite/ConvertCSVToAvro.java | 298 --
.../nifi/processors/kite/ConvertJSONToAvro.java | 202 -
.../nifi/processors/kite/FailureTracker.java | 83 -
.../processors/kite/StoreInKiteDataset.java | 162 -
.../org.apache.nifi.processor.Processor | 18 -
.../additionalDetails.html | 142 -
.../kite/TestAvroRecordConverter.java | 201 -
.../processors/kite/TestCSVToAvroProcessor.java | 217 -
.../kite/TestConfigurationProperty.java | 76 -
.../processors/kite/TestConvertAvroSchema.java | 216 -
.../nifi/processors/kite/TestGetSchema.java | 100 -
.../kite/TestJSONToAvroProcessor.java | 155 -
.../kite/TestKiteProcessorsCluster.java | 131 -
.../kite/TestKiteStorageProcessor.java | 170 -
.../apache/nifi/processors/kite/TestUtil.java | 103 -
nifi/nifi-nar-bundles/nifi-kite-bundle/pom.xml | 44 -
.../nifi-language-translation-nar/pom.xml | 36 -
.../src/main/resources/META-INF/NOTICE | 42 -
.../nifi-yandex-processors/.gitignore | 1 -
.../nifi-yandex-processors/pom.xml | 63 -
.../nifi/processors/yandex/YandexTranslate.java | 332 --
.../processors/yandex/model/Translation.java | 53 -
.../nifi/processors/yandex/util/Languages.java | 86 -
.../yandex/util/ObjectMapperResolver.java | 48 -
.../org.apache.nifi.processor.Processor | 16 -
.../processors/yandex/TestYandexTranslate.java | 223 -
.../nifi-language-translation-bundle/pom.xml | 48 -
.../nifi-mongodb-nar/pom.xml | 37 -
.../src/main/resources/META-INF/NOTICE | 24 -
.../nifi-mongodb-processors/pom.xml | 66 -
.../mongodb/AbstractMongoProcessor.java | 93 -
.../nifi/processors/mongodb/GetMongo.java | 184 -
.../nifi/processors/mongodb/PutMongo.java | 215 -
.../org.apache.nifi.processor.Processor | 16 -
.../nifi/processors/mongodb/GetMongoTest.java | 203 -
.../nifi/processors/mongodb/PutMongoTest.java | 256 --
.../nifi-mongodb-bundle/pom.xml | 35 -
.../pom.xml | 54 -
.../nifi/provenance/IndexConfiguration.java | 398 --
.../PersistentProvenanceRepository.java | 2130 ---------
.../provenance/RepositoryConfiguration.java | 299 --
.../nifi/provenance/StandardRecordReader.java | 507 ---
.../nifi/provenance/StandardRecordWriter.java | 343 --
.../provenance/expiration/ExpirationAction.java | 35 -
.../expiration/FileRemovalAction.java | 65 -
.../provenance/lucene/DeleteIndexAction.java | 120 -
.../nifi/provenance/lucene/DocsReader.java | 192 -
.../nifi/provenance/lucene/FieldNames.java | 24 -
.../nifi/provenance/lucene/IndexManager.java | 512 ---
.../nifi/provenance/lucene/IndexSearch.java | 116 -
.../nifi/provenance/lucene/IndexingAction.java | 133 -
.../nifi/provenance/lucene/LineageQuery.java | 119 -
.../nifi/provenance/lucene/LuceneUtil.java | 163 -
.../provenance/serialization/RecordReader.java | 103 -
.../provenance/serialization/RecordReaders.java | 123 -
.../provenance/serialization/RecordWriter.java | 95 -
.../provenance/serialization/RecordWriters.java | 39 -
.../nifi/provenance/toc/StandardTocReader.java | 155 -
.../nifi/provenance/toc/StandardTocWriter.java | 120 -
.../apache/nifi/provenance/toc/TocReader.java | 71 -
.../org/apache/nifi/provenance/toc/TocUtil.java | 40 -
.../apache/nifi/provenance/toc/TocWriter.java | 53 -
...he.nifi.provenance.ProvenanceEventRepository | 15 -
.../TestPersistentProvenanceRepository.java | 1323 ------
.../TestStandardRecordReaderWriter.java | 189 -
.../org/apache/nifi/provenance/TestUtil.java | 82 -
.../provenance/toc/TestStandardTocReader.java | 118 -
.../provenance/toc/TestStandardTocWriter.java | 42 -
.../nifi-provenance-repository-nar/pom.xml | 34 -
.../src/main/resources/META-INF/NOTICE | 202 -
.../nifi-volatile-provenance-repository/pom.xml | 42 -
.../VolatileProvenanceRepository.java | 745 ----
...he.nifi.provenance.ProvenanceEventRepository | 15 -
.../TestVolatileProvenanceRepository.java | 178 -
.../src/test/resources/nifi.properties | 18 -
.../nifi-provenance-repository-bundle/pom.xml | 43 -
.../nifi-social-media-nar/pom.xml | 36 -
.../src/main/resources/META-INF/NOTICE | 57 -
.../nifi-twitter-processors/.gitignore | 1 -
.../nifi-twitter-processors/pom.xml | 58 -
.../nifi/processors/twitter/GetTwitter.java | 362 --
.../org.apache.nifi.processor.Processor | 16 -
.../nifi-social-media-bundle/pom.xml | 33 -
.../nifi-solr-bundle/nifi-solr-nar/pom.xml | 36 -
.../src/main/resources/META-INF/LICENSE | 212 -
.../src/main/resources/META-INF/NOTICE | 51 -
.../nifi-solr-processors/pom.xml | 117 -
.../apache/nifi/processors/solr/GetSolr.java | 323 --
.../processors/solr/PutSolrContentStream.java | 295 --
.../nifi/processors/solr/SolrProcessor.java | 145 -
.../org.apache.nifi.processor.Processor | 16 -
.../additionalDetails.html | 48 -
.../solr/EmbeddedSolrServerFactory.java | 85 -
.../nifi/processors/solr/TestGetSolr.java | 221 -
.../solr/TestPutSolrContentStream.java | 416 --
.../src/test/resources/log4j.properties | 14 -
.../src/test/resources/solr/solr.xml | 18 -
.../testCollection/conf/_rest_managed.json | 3 -
.../testCollection/conf/lang/stopwords_en.txt | 54 -
.../resources/testCollection/conf/protwords.txt | 21 -
.../resources/testCollection/conf/schema.xml | 21 -
.../testCollection/conf/solrconfig.xml | 20 -
.../resources/testCollection/conf/synonyms.txt | 29 -
.../resources/testCollection/core.properties | 1 -
.../testdata/test-csv-multiple-docs.csv | 2 -
.../testdata/test-custom-json-single-doc.json | 15 -
.../testdata/test-solr-json-multiple-docs.json | 18 -
.../testdata/test-xml-multiple-docs.xml | 18 -
nifi/nifi-nar-bundles/nifi-solr-bundle/pom.xml | 50 -
.../nifi-standard-content-viewer/pom.xml | 75 -
.../web/StandardContentViewerController.java | 100 -
.../src/main/resources/META-INF/NOTICE | 19 -
.../main/webapp/META-INF/nifi-content-viewer | 17 -
.../src/main/webapp/WEB-INF/jsp/codemirror.jsp | 50 -
.../src/main/webapp/WEB-INF/web.xml | 29 -
.../src/main/webapp/css/main.css | 20 -
.../nifi-standard-nar/pom.xml | 49 -
.../src/main/resources/META-INF/LICENSE | 321 --
.../src/main/resources/META-INF/NOTICE | 155 -
.../nifi-standard-prioritizers/pom.xml | 36 -
.../prioritizer/FirstInFirstOutPrioritizer.java | 37 -
.../NewestFlowFileFirstPrioritizer.java | 37 -
.../OldestFlowFileFirstPrioritizer.java | 37 -
.../PriorityAttributePrioritizer.java | 87 -
...org.apache.nifi.flowfile.FlowFilePrioritizer | 18 -
.../prioritizer/NewestFirstPrioritizerTest.java | 66 -
.../prioritizer/OldestFirstPrioritizerTest.java | 66 -
.../PriorityAttributePrioritizerTest.java | 118 -
.../nifi-standard-processors/pom.xml | 284 --
.../standard/AbstractJsonPathProcessor.java | 135 -
.../standard/Base64EncodeContent.java | 151 -
.../nifi/processors/standard/BinFiles.java | 351 --
.../processors/standard/CompressContent.java | 324 --
.../nifi/processors/standard/ControlRate.java | 397 --
.../standard/ConvertCharacterSet.java | 176 -
.../processors/standard/DetectDuplicate.java | 260 --
.../processors/standard/DistributeLoad.java | 484 --
.../processors/standard/DuplicateFlowFile.java | 81 -
.../nifi/processors/standard/EncodeContent.java | 259 --
.../processors/standard/EncryptContent.java | 304 --
.../processors/standard/EvaluateJsonPath.java | 323 --
.../standard/EvaluateRegularExpression.java | 286 --
.../nifi/processors/standard/EvaluateXPath.java | 450 --
.../processors/standard/EvaluateXQuery.java | 446 --
.../processors/standard/ExecuteProcess.java | 512 ---
.../nifi/processors/standard/ExecuteSQL.java | 157 -
.../standard/ExecuteStreamCommand.java | 390 --
.../nifi/processors/standard/ExtractText.java | 344 --
.../processors/standard/GenerateFlowFile.java | 167 -
.../apache/nifi/processors/standard/GetFTP.java | 93 -
.../nifi/processors/standard/GetFile.java | 475 --
.../processors/standard/GetFileTransfer.java | 305 --
.../nifi/processors/standard/GetHTTP.java | 533 ---
.../nifi/processors/standard/GetJMSQueue.java | 77 -
.../nifi/processors/standard/GetJMSTopic.java | 348 --
.../nifi/processors/standard/GetSFTP.java | 109 -
.../processors/standard/HandleHttpRequest.java | 639 ---
.../processors/standard/HandleHttpResponse.java | 186 -
.../nifi/processors/standard/HashAttribute.java | 252 --
.../nifi/processors/standard/HashContent.java | 159 -
.../processors/standard/IdentifyMimeType.java | 159 -
.../nifi/processors/standard/InvokeHTTP.java | 749 ----
.../nifi/processors/standard/JmsConsumer.java | 239 -
.../nifi/processors/standard/ListenHTTP.java | 321 --
.../nifi/processors/standard/ListenUDP.java | 594 ---
.../nifi/processors/standard/LogAttribute.java | 265 --
.../nifi/processors/standard/MergeContent.java | 1075 -----
.../nifi/processors/standard/ModifyBytes.java | 131 -
.../processors/standard/MonitorActivity.java | 224 -
.../nifi/processors/standard/PostHTTP.java | 966 ----
.../nifi/processors/standard/PutEmail.java | 436 --
.../apache/nifi/processors/standard/PutFTP.java | 161 -
.../nifi/processors/standard/PutFile.java | 382 --
.../processors/standard/PutFileTransfer.java | 313 --
.../apache/nifi/processors/standard/PutJMS.java | 374 --
.../nifi/processors/standard/PutSFTP.java | 90 -
.../nifi/processors/standard/ReplaceText.java | 307 --
.../standard/ReplaceTextWithMapping.java | 388 --
.../processors/standard/RouteOnAttribute.java | 272 --
.../processors/standard/RouteOnContent.java | 241 -
.../nifi/processors/standard/ScanAttribute.java | 238 -
.../nifi/processors/standard/ScanContent.java | 305 --
.../processors/standard/SegmentContent.java | 188 -
.../nifi/processors/standard/SplitContent.java | 333 --
.../nifi/processors/standard/SplitJson.java | 200 -
.../nifi/processors/standard/SplitText.java | 374 --
.../nifi/processors/standard/SplitXml.java | 310 --
.../nifi/processors/standard/TransformXml.java | 204 -
.../nifi/processors/standard/UnpackContent.java | 442 --
.../nifi/processors/standard/ValidateXml.java | 153 -
.../servlets/ContentAcknowledgmentServlet.java | 133 -
.../standard/servlets/ListenHTTPServlet.java | 322 --
.../nifi/processors/standard/util/Bin.java | 162 -
.../processors/standard/util/BinManager.java | 235 -
.../standard/util/DocumentReaderCallback.java | 65 -
.../processors/standard/util/FTPTransfer.java | 586 ---
.../nifi/processors/standard/util/FTPUtils.java | 300 --
.../nifi/processors/standard/util/FileInfo.java | 167 -
.../processors/standard/util/FileTransfer.java | 259 --
.../standard/util/FlowFileSessionWrapper.java | 44 -
.../processors/standard/util/JdbcCommon.java | 165 -
.../processors/standard/util/JmsFactory.java | 434 --
.../standard/util/JmsProcessingSummary.java | 81 -
.../processors/standard/util/JmsProperties.java | 180 -
.../util/JsonPathExpressionValidator.java | 503 ---
.../standard/util/NLKBufferedReader.java | 181 -
.../standard/util/OpenPGPKeyBasedEncryptor.java | 277 --
.../util/OpenPGPPasswordBasedEncryptor.java | 175 -
.../standard/util/PasswordBasedEncryptor.java | 155 -
.../standard/util/SFTPConnection.java | 73 -
.../processors/standard/util/SFTPTransfer.java | 623 ---
.../processors/standard/util/SFTPUtils.java | 312 --
.../standard/util/SocksProxySocketFactory.java | 69 -
.../standard/util/UDPStreamConsumer.java | 216 -
.../util/ValidatingBase32InputStream.java | 76 -
.../util/ValidatingBase64InputStream.java | 75 -
.../standard/util/WrappedMessageConsumer.java | 77 -
.../standard/util/WrappedMessageProducer.java | 77 -
.../standard/util/XmlElementNotifier.java | 22 -
.../standard/util/XmlSplitterSaxParser.java | 112 -
.../org.apache.nifi.processor.Processor | 69 -
.../additionalDetails.html | 30 -
.../additionalDetails.html | 31 -
.../additionalDetails.html | 149 -
.../additionalDetails.html | 43 -
.../additionalDetails.html | 44 -
.../additionalDetails.html | 59 -
.../additionalDetails.html | 46 -
.../org/apache/tika/mime/custom-mimetypes.xml | 97 -
.../src/test/java/TestDynamicEnvironment.java | 29 -
.../src/test/java/TestIngestAndUpdate.java | 34 -
.../src/test/java/TestSuccess.java | 24 -
.../processors/standard/CaptureServlet.java | 55 -
.../processors/standard/HelloWorldServlet.java | 42 -
.../standard/RESTServiceContentModified.java | 78 -
.../standard/TestBase64EncodeContent.java | 82 -
.../standard/TestCompressContent.java | 179 -
.../processors/standard/TestControlRate.java | 67 -
.../standard/TestConvertCharacterSet.java | 44 -
.../standard/TestDetectDuplicate.java | 215 -
.../processors/standard/TestDistributeLoad.java | 138 -
.../standard/TestDuplicateFlowFile.java | 36 -
.../processors/standard/TestEncodeContent.java | 164 -
.../processors/standard/TestEncryptContent.java | 163 -
.../standard/TestEvaluateJsonPath.java | 366 --
.../processors/standard/TestEvaluateXPath.java | 158 -
.../processors/standard/TestEvaluateXQuery.java | 651 ---
.../processors/standard/TestExecuteProcess.java | 160 -
.../processors/standard/TestExecuteSQL.java | 169 -
.../standard/TestExecuteStreamCommand.java | 232 -
.../processors/standard/TestExtractText.java | 313 --
.../nifi/processors/standard/TestGetFile.java | 186 -
.../nifi/processors/standard/TestGetHTTP.java | 340 --
.../processors/standard/TestGetJMSQueue.java | 140 -
.../standard/TestHandleHttpRequest.java | 128 -
.../standard/TestHandleHttpResponse.java | 180 -
.../processors/standard/TestHashAttribute.java | 99 -
.../processors/standard/TestHashContent.java | 66 -
.../standard/TestIdentifyMimeType.java | 106 -
.../processors/standard/TestInvokeHTTP.java | 672 ---
.../processors/standard/TestJmsConsumer.java | 171 -
.../nifi/processors/standard/TestListenUDP.java | 213 -
.../processors/standard/TestMergeContent.java | 846 ----
.../processors/standard/TestModifyBytes.java | 189 -
.../standard/TestMonitorActivity.java | 191 -
.../nifi/processors/standard/TestPostHTTP.java | 248 --
.../nifi/processors/standard/TestPutEmail.java | 82 -
.../processors/standard/TestReplaceText.java | 389 --
.../standard/TestReplaceTextLineByLine.java | 336 --
.../standard/TestReplaceTextWithMapping.java | 332 --
.../standard/TestRouteOnAttribute.java | 160 -
.../processors/standard/TestRouteOnContent.java | 72 -
.../processors/standard/TestScanAttribute.java | 148 -
.../processors/standard/TestScanContent.java | 103 -
.../processors/standard/TestSegmentContent.java | 64 -
.../nifi/processors/standard/TestServer.java | 159 -
.../processors/standard/TestSplitContent.java | 353 --
.../nifi/processors/standard/TestSplitJson.java | 214 -
.../nifi/processors/standard/TestSplitText.java | 163 -
.../nifi/processors/standard/TestSplitXml.java | 102 -
.../processors/standard/TestTransformXml.java | 115 -
.../processors/standard/TestUnpackContent.java | 224 -
.../processors/standard/TestValidateXml.java | 41 -
.../standard/UserAgentTestingServlet.java | 41 -
.../standard/util/TestJdbcCommon.java | 155 -
.../standard/util/TestJdbcHugeStream.java | 267 --
.../CharacterSetConversionSamples/Converted.txt | 1 -
.../Converted2.txt | Bin 134580 -> 0 bytes
.../CharacterSetConversionSamples/Original.txt | 418 --
.../resources/CompressedData/SampleFile.txt | 418 --
.../resources/CompressedData/SampleFile.txt.bz2 | Bin 388 -> 0 bytes
.../resources/CompressedData/SampleFile.txt.gz | Bin 321 -> 0 bytes
.../CompressedData/SampleFile1.txt.bz2 | Bin 154 -> 0 bytes
.../resources/CompressedData/SampleFile1.txt.gz | Bin 321 -> 0 bytes
.../CompressedData/SampleFileConcat.txt | 835 ----
.../CompressedData/SampleFileConcat.txt.bz2 | Bin 542 -> 0 bytes
.../test/resources/ExecuteCommand/1000bytes.txt | 1 -
.../ExecuteCommand/TestDynamicEnvironment.jar | Bin 1200 -> 0 bytes
.../ExecuteCommand/TestIngestAndUpdate.jar | Bin 1170 -> 0 bytes
.../resources/ExecuteCommand/TestSuccess.jar | Bin 827 -> 0 bytes
.../src/test/resources/ExecuteCommand/test.txt | 2 -
.../dictionary-with-empty-new-lines | 5 -
.../ScanAttribute/dictionary-with-extra-info | 9 -
.../test/resources/ScanAttribute/dictionary1 | 5 -
.../test/resources/TestEncryptContent/text.txt | 17 -
.../resources/TestEncryptContent/text.txt.asc | 33 -
.../test/resources/TestIdentifyMimeType/1.7z | Bin 133 -> 0 bytes
.../test/resources/TestIdentifyMimeType/1.jar | Bin 466 -> 0 bytes
.../test/resources/TestIdentifyMimeType/1.mdb | Bin 237568 -> 0 bytes
.../test/resources/TestIdentifyMimeType/1.pdf | Bin 277032 -> 0 bytes
.../test/resources/TestIdentifyMimeType/1.tar | Bin 10240 -> 0 bytes
.../resources/TestIdentifyMimeType/1.tar.gz | Bin 154 -> 0 bytes
.../test/resources/TestIdentifyMimeType/1.txt | 1 -
.../resources/TestIdentifyMimeType/1.txt.bz2 | Bin 59 -> 0 bytes
.../resources/TestIdentifyMimeType/1.txt.gz | Bin 50 -> 0 bytes
.../test/resources/TestIdentifyMimeType/1.xml | 20 -
.../test/resources/TestIdentifyMimeType/1.zip | Bin 165 -> 0 bytes
.../TestIdentifyMimeType/bgBannerFoot.png | Bin 189 -> 0 bytes
.../TestIdentifyMimeType/blueBtnBg.jpg | Bin 356 -> 0 bytes
.../TestIdentifyMimeType/flowfilev1.tar | Bin 10240 -> 0 bytes
.../resources/TestIdentifyMimeType/flowfilev3 | Bin 40 -> 0 bytes
.../resources/TestIdentifyMimeType/grid.gif | Bin 135 -> 0 bytes
.../resources/TestJson/control-characters.json | 7 -
.../test/resources/TestJson/json-sample.json | 415 --
.../test/resources/TestMergeContent/demarcate | 1 -
.../src/test/resources/TestMergeContent/foot | 1 -
.../src/test/resources/TestMergeContent/head | 1 -
.../test/resources/TestMergeContent/place.avsc | 7 -
.../test/resources/TestMergeContent/user.avsc | 9 -
.../test/resources/TestModifyBytes/noFooter.txt | 10 -
.../TestModifyBytes/noFooter_noHeader.txt | 10 -
.../test/resources/TestModifyBytes/noHeader.txt | 11 -
.../test/resources/TestModifyBytes/testFile.txt | 11 -
.../TestReplaceTextLineByLine/$1$1.txt | 11 -
.../BRue_cRue_RiRey.txt | 11 -
.../TestReplaceTextLineByLine/Blu$2e_clu$2e.txt | 11 -
.../TestReplaceTextLineByLine/D$d_h$d.txt | 11 -
.../TestReplaceTextLineByLine/Good.txt | 1 -
.../TestReplaceTextLineByLine/Spider.txt | 11 -
.../TestReplaceTextLineByLine/[DODO].txt | 11 -
.../TestReplaceTextLineByLine/cu[$1]_Po[$1].txt | 11 -
.../TestReplaceTextLineByLine/cu_Po.txt | 11 -
.../TestReplaceTextLineByLine/food.txt | 11 -
.../TestReplaceTextLineByLine/testFile.txt | 11 -
.../color-fruit-backreference-mapping.txt | 7 -
.../color-fruit-blank-mapping.txt | 7 -
.../color-fruit-escaped-dollar-mapping.txt | 7 -
...t-excessive-backreference-mapping-simple.txt | 6 -
...or-fruit-excessive-backreference-mapping.txt | 6 -
...olor-fruit-invalid-backreference-mapping.txt | 7 -
.../color-fruit-mapping.txt | 7 -
.../color-fruit-no-match-mapping.txt | 7 -
.../color-fruit-space-mapping.txt | 6 -
.../colors-without-dashes.txt | 4 -
.../TestReplaceTextWithMapping/colors.txt | 4 -
.../test/resources/TestScanContent/helloWorld | 1 -
.../resources/TestScanContent/wellthengood-bye | 1 -
.../src/test/resources/TestSplitText/1.txt | 5 -
.../src/test/resources/TestSplitText/2.txt | 5 -
.../src/test/resources/TestSplitText/3.txt | 5 -
.../src/test/resources/TestSplitText/4.txt | 3 -
.../src/test/resources/TestSplitText/5.txt | 7 -
.../src/test/resources/TestSplitText/6.txt | 7 -
.../test/resources/TestSplitText/original.txt | 12 -
.../test/resources/TestTransformXml/math.html | 8 -
.../test/resources/TestTransformXml/math.xml | 21 -
.../test/resources/TestTransformXml/math.xsl | 36 -
.../test/resources/TestTransformXml/tokens.csv | 2 -
.../test/resources/TestTransformXml/tokens.xml | 17 -
.../test/resources/TestTransformXml/tokens.xsl | 103 -
.../resources/TestUnpackContent/data.flowfilev2 | Bin 255 -> 0 bytes
.../resources/TestUnpackContent/data.flowfilev3 | Bin 357 -> 0 bytes
.../test/resources/TestUnpackContent/data.tar | Bin 3584 -> 0 bytes
.../test/resources/TestUnpackContent/data.zip | Bin 359 -> 0 bytes
.../resources/TestUnpackContent/folder/cal.txt | 8 -
.../resources/TestUnpackContent/folder/date.txt | 1 -
.../src/test/resources/TestXml/XmlBundle.xsd | 34 -
.../src/test/resources/TestXml/fruit.xml | 47 -
.../src/test/resources/TestXml/subNode.xml | 21 -
.../src/test/resources/TestXml/xml-bundle-1 | 51 -
.../src/test/resources/TestXml/xml-snippet.xml | 25 -
.../src/test/resources/hello.txt | 1 -
.../src/test/resources/localhost-ks.jks | Bin 1298 -> 0 bytes
.../src/test/resources/localhost-ts.jks | Bin 891 -> 0 bytes
.../src/test/resources/localhost.cer | Bin 829 -> 0 bytes
.../src/test/resources/logback-test.xml | 49 -
.../nifi-standard-reporting-tasks/pom.xml | 60 -
.../ControllerStatusReportingTask.java | 343 --
.../nifi/controller/MonitorDiskUsage.java | 118 -
.../apache/nifi/controller/MonitorMemory.java | 257 --
.../ganglia/StandardGangliaReporter.java | 269 --
.../org.apache.nifi.reporting.ReportingTask | 18 -
.../additionalDetails.html | 70 -
.../additionalDetails.html | 41 -
.../nifi/controller/TestMonitorDiskUsage.java | 61 -
.../nifi-standard-bundle/pom.xml | 65 -
.../nifi-dbcp-service-api/pom.xml | 31 -
.../java/org/apache/nifi/dbcp/DBCPService.java | 34 -
.../nifi-dbcp-service-nar/pom.xml | 37 -
.../src/main/resources/META-INF/LICENSE | 239 -
.../src/main/resources/META-INF/NOTICE | 35 -
.../nifi-dbcp-service/pom.xml | 61 -
.../apache/nifi/dbcp/DBCPConnectionPool.java | 239 -
.../nifi/dbcp/DatabaseSystemDescriptor.java | 51 -
.../org/apache/nifi/dbcp/DatabaseSystems.java | 83 -
.../java/org/apache/nifi/dbcp/DriverShim.java | 74 -
...org.apache.nifi.controller.ControllerService | 15 -
.../org/apache/nifi/dbcp/DBCPServiceTest.java | 289 --
.../apache/nifi/dbcp/TestDatabaseSystems.java | 31 -
.../org/apache/nifi/dbcp/TestProcessor.java | 45 -
.../nifi-dbcp-service-bundle/pom.xml | 29 -
.../pom.xml | 31 -
.../distributed/cache/client/Deserializer.java | 41 -
.../cache/client/DistributedMapCacheClient.java | 140 -
.../cache/client/DistributedSetCacheClient.java | 78 -
.../distributed/cache/client/Serializer.java | 41 -
.../exception/DeserializationException.java | 33 -
.../exception/SerializationException.java | 33 -
.../pom.xml | 51 -
.../distributed/cache/client/CommsSession.java | 46 -
.../DistributedMapCacheClientService.java | 331 --
.../DistributedSetCacheClientService.java | 220 -
.../cache/client/SSLCommsSession.java | 109 -
.../cache/client/StandardCommsSession.java | 125 -
...org.apache.nifi.controller.ControllerService | 16 -
.../nifi-distributed-cache-protocol/pom.xml | 33 -
.../cache/protocol/ProtocolHandshake.java | 116 -
.../protocol/exception/HandshakeException.java | 28 -
.../nifi-distributed-cache-server/pom.xml | 70 -
.../cache/server/AbstractCacheServer.java | 204 -
.../distributed/cache/server/CacheRecord.java | 57 -
.../distributed/cache/server/CacheServer.java | 29 -
.../cache/server/DistributedCacheServer.java | 114 -
.../cache/server/DistributedSetCacheServer.java | 73 -
.../cache/server/EvictionPolicy.java | 77 -
.../cache/server/SetCacheServer.java | 105 -
.../server/map/DistributedMapCacheServer.java | 78 -
.../distributed/cache/server/map/MapCache.java | 35 -
.../cache/server/map/MapCacheRecord.java | 59 -
.../cache/server/map/MapCacheServer.java | 153 -
.../cache/server/map/MapPutResult.java | 60 -
.../cache/server/map/PersistentMapCache.java | 232 -
.../cache/server/map/SimpleMapCache.java | 188 -
.../cache/server/set/PersistentSetCache.java | 195 -
.../distributed/cache/server/set/SetCache.java | 32 -
.../cache/server/set/SetCacheRecord.java | 56 -
.../cache/server/set/SetCacheResult.java | 42 -
.../cache/server/set/SimpleSetCache.java | 118 -
...org.apache.nifi.controller.ControllerService | 16 -
.../cache/server/TestServerAndClient.java | 536 ---
.../src/test/resources/localhost-ks.jks | Bin 1298 -> 0 bytes
.../src/test/resources/localhost-ts.jks | Bin 891 -> 0 bytes
.../nifi-distributed-cache-services-nar/pom.xml | 43 -
.../src/main/resources/META-INF/NOTICE | 19 -
.../pom.xml | 30 -
.../nifi-http-context-map-api/pom.xml | 37 -
.../org/apache/nifi/http/HttpContextMap.java | 69 -
.../nifi-http-context-map-nar/pom.xml | 37 -
.../src/main/resources/META-INF/NOTICE | 19 -
.../nifi-http-context-map/pom.xml | 44 -
.../nifi/http/StandardHttpContextMap.java | 188 -
...org.apache.nifi.controller.ControllerService | 15 -
.../nifi-http-context-map-bundle/pom.xml | 31 -
.../nifi-load-distribution-service-api/pom.xml | 30 -
.../nifi/loading/LoadDistributionListener.java | 24 -
.../nifi/loading/LoadDistributionService.java | 33 -
.../nifi-ssl-context-nar/pom.xml | 35 -
.../src/main/resources/META-INF/NOTICE | 19 -
.../nifi-ssl-context-service/pom.xml | 47 -
.../nifi/ssl/StandardSSLContextService.java | 446 --
...org.apache.nifi.controller.ControllerService | 15 -
.../apache/nifi/ssl/SSLContextServiceTest.java | 163 -
.../java/org/apache/nifi/ssl/TestProcessor.java | 47 -
.../src/test/resources/localhost-ks.jks | Bin 1298 -> 0 bytes
.../src/test/resources/localhost-ts.jks | Bin 891 -> 0 bytes
.../nifi-ssl-context-bundle/pom.xml | 28 -
.../nifi-ssl-context-service-api/pom.xml | 30 -
.../org/apache/nifi/ssl/SSLContextService.java | 61 -
.../nifi-standard-services-api-nar/pom.xml | 51 -
.../nifi-standard-services/pom.xml | 37 -
.../nifi-update-attribute-model/pom.xml | 26 -
.../apache/nifi/update/attributes/Action.java | 55 -
.../nifi/update/attributes/Condition.java | 49 -
.../apache/nifi/update/attributes/Criteria.java | 141 -
.../nifi/update/attributes/FlowFilePolicy.java | 34 -
.../org/apache/nifi/update/attributes/Rule.java | 65 -
.../update/attributes/serde/CriteriaSerDe.java | 128 -
.../nifi-update-attribute-nar/pom.xml | 43 -
.../src/main/resources/META-INF/NOTICE | 19 -
.../nifi-update-attribute-processor/pom.xml | 53 -
.../processors/attributes/UpdateAttribute.java | 487 --
.../org.apache.nifi.processor.Processor | 15 -
.../additionalDetails.html | 253 --
.../update/attributes/TestUpdateAttribute.java | 425 --
.../nb-configuration.xml | 18 -
.../nifi-update-attribute-ui/pom.xml | 95 -
.../attributes/UpdateAttributeModelFactory.java | 127 -
.../attributes/api/ObjectMapperResolver.java | 51 -
.../update/attributes/api/RuleResource.java | 687 ---
.../nifi/update/attributes/dto/ActionDTO.java | 68 -
.../update/attributes/dto/ConditionDTO.java | 59 -
.../nifi/update/attributes/dto/DtoFactory.java | 68 -
.../nifi/update/attributes/dto/RuleDTO.java | 65 -
.../update/attributes/entity/ActionEntity.java | 65 -
.../attributes/entity/ConditionEntity.java | 65 -
.../entity/EvaluationContextEntity.java | 74 -
.../update/attributes/entity/RuleEntity.java | 65 -
.../update/attributes/entity/RulesEntity.java | 66 -
.../src/main/resources/META-INF/LICENSE | 238 -
.../src/main/resources/META-INF/NOTICE | 49 -
.../META-INF/nifi-processor-configuration | 15 -
.../src/main/webapp/WEB-INF/jsp/worksheet.jsp | 188 -
.../src/main/webapp/WEB-INF/web.xml | 46 -
.../src/main/webapp/css/main.css | 426 --
.../src/main/webapp/images/bgInputText.png | Bin 139 -> 0 bytes
.../src/main/webapp/images/buttonNew.png | Bin 590 -> 0 bytes
.../src/main/webapp/images/iconDelete.png | Bin 670 -> 0 bytes
.../src/main/webapp/images/iconInfo.png | Bin 550 -> 0 bytes
.../src/main/webapp/js/application.js | 1910 --------
.../nifi-update-attribute-bundle/pom.xml | 52 -
nifi/nifi-nar-bundles/pom.xml | 130 -
nifi/pom.xml | 931 ----
pom.xml | 1284 ++++++
5697 files changed, 381689 insertions(+), 387105 deletions(-)
----------------------------------------------------------------------
[14/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/asciidoc/user-guide.adoc
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/user-guide.adoc b/nifi-docs/src/main/asciidoc/user-guide.adoc
new file mode 100644
index 0000000..b793af4
--- /dev/null
+++ b/nifi-docs/src/main/asciidoc/user-guide.adoc
@@ -0,0 +1,1468 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+Apache NiFi User Guide
+======================
+Apache NiFi Team <de...@nifi.apache.org>
+:homepage: http://nifi.apache.org
+
+
+Introduction
+------------
+Apache NiFi is a dataflow system based on the concepts of flow-based programming. It supports
+powerful and scalable directed graphs of data routing, transformation, and system mediation logic. NiFi has
+a web-based user interface for design, control, feedback, and monitoring of dataflows. It is highly configurable
+along several dimensions of quality of service, such as loss-tolerant versus guaranteed delivery, low latency versus
+high throughput, and priority-based queuing. NiFi provides fine-grained data provenance for all data received, forked, joined
+cloned, modified, sent, and ultimately dropped upon reaching its configured end-state.
+
+See the Admin Guide for information about system requirements, installation, and configuration. Once NiFi is installed,
+use a supported web browser to view the User Interface. Supported web browsers include:
+
+* Internet Explorer 9+ (see note below)
+* Mozilla FireFox 24+
+* Google Chrome 36+
+* Safari 8
+
+Note that there is a known issue in Internet Explorer (IE) 10 and 11 that can cause problems when moving items on the NiFi graph. If you encounter this problem, we suggest using a browser other than IE. This known issue is described here: https://connect.microsoft.com/IE/Feedback/Details/1050422.
+
+
+
+[template="glossary", id="terminology"]
+Terminology
+-----------
+*DataFlow Manager*: A DataFlow Manager (DFM) is a NiFi user who has permissions to add, remove, and modify components of a NiFi dataflow.
+
+*FlowFile*: The FlowFile represents a single piece of data in NiFi. A FlowFile is made up of two components:
+ FlowFile Attributes and FlowFile Content.
+ Content is the data that is represented by the FlowFile. Attributes are characteristics that provide information or
+ context about the data; they are made up of key-value pairs.
+ All FlowFiles have the following Standard Attributes:
+
+- *uuid*: A unique identifier for the FlowFile
+- *filename*: A human-readable filename that may be used when storing the data to disk or in an external service
+- *path*: A hierarchically structured value that can be used when storing data to disk or an external service so that the data is not stored in a single directory
+
+*Processor*: The Processor is the NiFi component that is used to listen for incoming data; pull data from external sources;
+ publish data to external sources; and route, transform, or extract information from FlowFiles.
+
+*Relationship*: Each Processor has zero or more Relationships defined for it. These Relationships are named to indicate the result of processing a FlowFile.
+ After a Processor has finished processing a FlowFile, it will route (or “transfer”) the FlowFile to one of the Relationships.
+ A DFM is then able to connect each of these Relationships to other components in order to specify where the FlowFile should
+ go next under each potential processing result.
+
+*Connection*: A DFM creates an automated dataflow by dragging components from the Components part of the NiFi toolbar to the canvas
+ and then connecting the components together via Connections. Each connection consists of one or more Relationships.
+ For each Connection that is drawn, a DFM can determine which Relationships should be used for the Connection.
+ This allows data to be routed in different ways based on its processing outcome. Each connection houses a FlowFile Queue.
+ When a FlowFile is transferred to a particular Relationship, it is added to the queue belonging to the associated Connection.
+
+*Controller Service*: Controller Services are extension points that, after being added and configured by a DFM in the User Interface, will start up when NiFi starts up and provide information for use by other components (such as processors or other controller services). A common Controller Service used by several components is the StandardSSLContextService. It provides the ability to configure keystore and/or truststore properties once and reuse that configuration throughout the application. The idea is that, rather than configure this information in every processor that might need it, the controller service provides it for any processor to use as needed.
+
+*Reporting Task*: Reporting Tasks run in the background to provide statistical reports about what is happening in the NiFi instance. The DFM adds and configures Reporting Tasks in the User Interface as desired. Common reporting tasks include the ControllerStatusReportingTask, MonitorDiskUsage reporting task, MonitorMemory reporting task, and the StandardGangliaReporter.
+
+*Funnel*: A funnel is a NiFi component that is used to combine the data from several Connections into a single Connection.
+
+*Process Group*: When a dataflow becomes complex, it often is beneficial to reason about the dataflow at a higher, more abstract level.
+ NiFi allows multiple components, such as Processors, to be grouped together into a Process Group.
+ The NiFi User Interface then makes it easy for a DFM to connect together multiple Process Groups into a logical dataflow,
+ as well as allowing the DFM to enter a Process Group in order to see and manipulate the components within the Process Group.
+
+*Port*: Dataflows that are constructed using one or more Process Groups need a way to connect a Process Group to other dataflow components.
+ This is achieved by using Ports. A DFM can add any number of Input Ports and Output Ports to a Process Group and name these ports appropriately.
+
+*Remote Process Group*: Just as data is transferred into and out of a Process Group, it is sometimes necessary to transfer data from one instance of NiFi to another.
+ While NiFi provides many different mechanisms for transferring data from one system to another, Remote Process Groups are often the easiest way to accomplish
+ this if transferring data to another instance of NiFi.
+
+*Bulletin*: The NiFi User Interface provides a significant amount of monitoring and feedback about the current status of the application.
+ In addition to rolling statistics and the current status provided for each component, components are able to report Bulletins.
+ Whenever a component reports a Bulletin, a bulletin icon is displayed on that component. System-level bulletins are displayed on the Status bar near the top of the page.
+ Using the mouse to hover over that icon will provide a tool-tip that shows the time and severity (Debug, Info, Warning, Error) of the Bulletin,
+ as well as the message of the Bulletin.
+ Bulletins from all components can also be viewed and filtered in the Bulletin Board Page, available in the Management Toolbar.
+
+*Template*: Often times, a dataflow is comprised of many sub-flows that could be reused. NiFi allows DFMs to select a part of the dataflow
+ (or the entire dataflow) and create a Template. This Template is given a name and can then be dragged onto the canvas just like the other components.
+ As a result, several components may be combined together to make a larger building block from which to create a dataflow.
+ These templates can also be exported as XML and imported into another NiFi instance, allowing these building blocks to be shared.
+
+
+[[User_Interface]]
+NiFi User Interface
+-------------------
+
+The NiFi User Interface (UI) provides mechanisms for creating automated dataflows, as well as visualizing,
+editing, monitoring, and administering those dataflows. The UI can be broken down into several segments,
+each responsible for different functionality of the application. This section provides screenshots of the
+application and highlights the different segments of the UI. Each segment is discussed in further detail later
+in the document.
+
+When the application is started, the user is able to navigate to the User Interface by going to the default address of
+`http://<hostname>:8080/nifi` in a web browser. There are no permissions configured by default, so anyone is
+able to view and modify the dataflow. For information on securing the system, see the Systems Administrator guide.
+
+When a DFM navigates to the UI for the first time, a blank canvas is provided on which a dataflow can be built:
+
+image::new-flow.png["Empty Flow"]
+
+Along the top of the of the screen is a toolbar that contains several of these segments.
+To the left is the Components Toolbar. This toolbar consists of the different components that can be dragged onto the canvas.
+
+Next to the Components Toolbar is the Actions Toolbar. This toolbar consists of buttons to manipulate the existing
+components on the graph. To the right of the Actions Toolbar is the Search Toolbar. This toolbar consists of a single
+Search field that allows users to easily find components on the graph. Users are able to search by component name,
+type, identifier, configuration properties, and their values.
+
+The Management Toolbar sits to the right-hand side of the screen. This toolbar consists of buttons that are
+used by DFMs to manage the flow as well as by administrators who manage user access
+and configure system properties, such as how many system resources should be provided to the application.
+
+image::nifi-toolbar-components.png["NiFi Components Toolbar"]
+
+Next, we have segments that provide capabilities to easily navigate around the graph. On the left-hand side is a toolbar that
+provides the ability to pan around the graph and zoom in and out. On the right-hand side is a “Birds-Eye View” of the dataflow.
+This provides a high-level view of the dataflow and allows the user to quickly and easily pan across large portions of the dataflow.
+Along the top of the screen is a trail of breadcrumbs. As users navigate into and out of Process Groups, the breadcrumbs show
+the depth in the flow and each Process Group that was entered to reach this depth. Each of the Process Groups listed in the breadcrumbs
+is a link that will take you back up to that level in the flow.
+
+image::nifi-navigation.png["NiFi Navigation"]
+
+[[status_bar]]
+Below the breadcrumbs lives the Status bar. The Status bar provides information about how many Processors exist in the graph in
+each state (Stopped, Running, Invalid, Disabled), how many Remote Process Groups exist on the graph in each state
+(Transmitting, Not Transmitting), the number of threads that are currently active in the flow, the amount of data that currently
+exists in the flow, and the timestamp at which all of this information was last refreshed. If there are any System-Level bulletins,
+these are shown in the Status bar as well. Additionally, if the instance of NiFi is clustered, the Status bar shows how many nodes
+are in the cluster and how many are currently connected.
+
+image::status-bar.png["NiFi Status Bar"]
+
+
+
+
+
+Building a DataFlow
+-------------------
+
+A DFM is able to build an automated dataflow using the NiFi User Interface (UI). Simply drag components from the toolbar to the canvas, configure the components to meet specific needs, and connect
+the components together.
+
+
+=== Adding Components to the Canvas
+
+In the User Interface section above outlined the different segments of the UI and pointed out a Components Toolbar.
+This section looks at each of the Components in that toolbar:
+
+image::components.png["Components"]
+
+[[processor]]
+image:iconProcessor.png["Processor", width=32]
+*Processor*: The Processor is the most commonly used component, as it is responsible for data ingress, egress, routing, and
+ manipulating. There are many different types of Processors. In fact, this is a very common Extension Point in NiFi,
+ meaning that many vendors may implement their own Processors to perform whatever functions are necessary for their use case.
+ When a Processor is dragged onto the graph, the user is presented with a dialog to choose which type of Processor to use:
+
+image::add-processor.png["Add Processor Dialog"]
+
+In the top-right corner, the user is able to filter the list based on the Processor Type or the Tags associated with a Processor.
+Processor developers have the ability to add Tags to their Processors. These tags are used in this dialog for filtering and are
+displayed on the left-hand side in a Tag Cloud. The more Processors that exist with a particular Tag, the larger the Tag appears
+in the Tag Cloud. Clicking a Tag in the Cloud will filter the available Processors to only those that contain that Tag. If multiple
+Tags are selected, only those Processors that contain all of those Tags are shown. For example, if we want to show only those
+Processors that allow us to ingest data via HTTP, we can select both the `http` Tag and the `ingest` Tag:
+
+image::add-processor-with-tag-cloud.png["Add Processor with Tag Cloud"]
+
+Clicking the `Add` button or double-clicking on a Processor Type will add the selected Processor to the canvas at the
+location that it was dropped.
+
+*Note*: For any component added to the graph, it is possible to select it with the mouse and move it anywhere on the graph. Also, it is possible to select multiple items at once by either holding down the Shift key and selecting each item or by holding down the Shift key and dragging a selection box around the desired components.
+
+
+[[input_port]]
+image:iconInputPort.png["Input Port", width=32]
+*Input Port*: Input Ports provide a mechanism for transferring data into a Process Group. When an Input Port is dragged
+onto the canvas, the DFM is prompted to name the Port. All Ports within a Process Group must have unique names.
+
+All components exist only within a Process Group. When a user initially navigates to the NiFi page, the user is placed
+in the Root Process Group. If the Input Port is dragged onto the Root Process Group, the Input Port provides a mechanism
+to receive data from remote instances of NiFi via <<site-to-site,Site-to-Site>>. In this case, the Input Port can be configured
+to restrict access to appropriate users, if NiFi is configured to run securely. For information on configuring NiFi to run
+securely, see the
+link:administration-guide.html[Admin Guide].
+
+
+
+[[output_port]]
+image:iconOutputPort.png["Output Port", width=32]
+*Output Port*: Output Ports provide a mechanism for transferring data from a Process Group to destinations outside
+of the Process Group. When an Output Port is dragged onto the canvas, the DFM is prompted to name the Port. All Ports
+within a Process Group must have unique names.
+
+If the Output Port is dragged onto the Root Process Group, the Output Port provides a mechanism for sending data to
+remote instances of NiFi via <<site-to-site,Site-to-Site>>. In this case, the Port acts as a queue. As remote instances
+of NiFi pull data from the port, that data is removed from the queues of the incoming Connections. If NiFi is configured
+to run securely, the Output Port can be configured to restrict access to appropriate users. For information on configuring
+NiFi to run securely, see the
+link:administration-guide.html[Admin Guide].
+
+
+[[process_group]]
+image:iconProcessGroup.png["Process Group", width=32]
+*Process Group*: Process Groups can be used to logically group a set of components so that the dataflow is easier to understand
+and maintain. When a Process Group is dragged onto the canvas, the DFM is prompted to name the Process Group. All Process
+Groups within the same parent group must have unique names. The Process Group will then be nested within that parent group.
+
+
+
+[[remote_process_group]]
+image:iconRemoteProcessGroup.png["Remote Process Group", width=32]
+*Remote Process Group*: Remote Process Groups appear and behave similar to Process Groups. However, the Remote Process Group (RPG)
+references a remote instance of NiFi. When an RPG is dragged onto the canvas, rather than being prompted for a name, the DFM
+is prompted for the URL of the remote NiFi instance. If the remote NiFi is a clustered instance, the URL that should be used
+is the URL of the remote instance's NiFi Cluster Manager (NCM). When data is transferred to a clustered instance of NiFi
+via an RPG, the RPG it will first connect to the remote instance's NCM to determine which nodes are in the cluster and
+how busy each node is. This information is then used to load balance the data that is pushed to each node. The remote NCM is
+then interrogated periodically to determine information about any nodes that are dropped from or added to the cluster and to
+recalculate the load balancing based on each node's load. For more information, see the section on <<site-to-site,Site-to-Site>>.
+
+
+[[funnel]]
+image:iconFunnel.png["Funnel", width=32]
+*Funnel*: Funnels are used to combine the data from many Connections into a single Connection. This has two advantages.
+First, if many Connections are created with the same destination, the canvas can become cluttered if those Connections
+have to span a large space. By funneling these Connections into a single Connection, that single Connection can then be
+drawn to span that large space instead. Secondly, Connections can be configured with FlowFile Prioritizers. Data from
+several Connections can be funneled into a single Connection, providing the ability to Prioritize all of the data on that
+one Connection, rather than prioritizing the data on each Connection independently.
+
+
+[[template]]
+image:iconTemplate.png["Template", width=32]
+*Template*: Templates can be created by DFMs from sections of the flow, or they can be imported from other
+dataflows. These Templates provide larger building blocks for creating a complex flow quickly. When the Template is
+dragged onto the canvas, the DFM is provided a dialog to choose which Template to add to the canvas:
+
+image::instantiate-template.png["Instantiate Template Dialog"]
+
+Clicking the drop-down box shows all available Templates. Any Template that was created with a description will show a question mark
+icon, indicating that there is more information. Hovering over the icon with the mouse will show this description:
+
+image::instantiate-template-description.png["Instantiate Template Dialog"]
+
+
+
+[[label]]
+image:iconLabel.png["Label"]
+*Label*: Labels are used to provide documentation to parts of a dataflow. When a Label is dropped onto the canvas,
+it is created with a default size. The Label can then be resized by dragging the handle in the bottom-right corner.
+The Label has no text when initially created. The text of the Label can be added by right-clicking on the Label and
+choosing `Configure...`
+
+
+
+
+=== Configuring a Processor
+
+Once a Processor has been dragged onto the Canvas, it is ready to configure. This is done by right-clicking on the
+Processor and clicking the `Configure...` option from the context menu. The configuration dialog is opened with four
+different tabs, each of which is discussed below. Once you have finished configuring the Processor, you can apply
+the changes by clicking the `Apply` button or cancel all changes by clicking the `Cancel` button.
+
+Note that after a Processor has been started, the context menu shown for the Processor no longer has a `Configure...`
+option but rather has a `View Configuration` option. Processor configuration cannot be changed while the Processor is
+running. You must first stop the Processor and wait for all of its active tasks to complete before configuring
+the Processor again.
+
+
+==== Settings Tab
+
+The first tab in the Processor Configuration dialog is the Settings tab:
+
+image::settings-tab.png["Settings Tab"]
+
+This tab contains several different configuration items. First, it allows the DFM to change the name of the Processor.
+The name of a Processor by default is the same as the Processor type. Next to the Processor Name is a checkbox, indicating
+ whether the Processor is Enabled. When a Processor is added to the graph, it is enabled. If the
+Processor is disabled, it cannot be started. The disabled state is used to indicate that when a group of Processors is started,
+such as when a DFM starts an entire Process Group, this (disabled) Processor should be excluded.
+
+Below the Name configuration, the Processor's unique identifier is displayed along with the Processor's type. These
+values cannot be modified.
+
+Next are two dialogues for configuring `Penalty duration' and `Yield duration'. During the normal course of processing a
+piece of data (a FlowFile), an event may occur that indicates that the data cannot be processed at this time but the
+data may be processable at a later time. When this occurs, the Processor may choose to Penalize the FlowFile. This will
+prevent the FlowFile from being Processed for some period of time. For example, if the Processor is to push the data
+to a remote service, but the remote service already has a file with the same name as the filename that the Processor
+is specifying, the Processor may penalize the FlowFile. The `Penalty duration' allows the DFM to specify how long the
+FlowFile should be penalized. The default value is 30 seconds.
+
+Similarly, the Processor may determine that some situation exists such that the Processor can no longer make any progress,
+regardless of the data that it is processing. For example, if a Processor is to push data to a remote service and that
+service is not responding, the Processor cannot make any progress. As a result, the Processor should `yield,' which will
+prevent the Processor from being scheduled to run for some period of time. That period of time is specified by setting
+the `Yield duration.' The default value is 1 second.
+
+The last configurable option on the left-hand side of the Settings tab is the Bulletin level. Whenever the Processor writes
+to its log, the Processor also will generate a Bulletin. This setting indicates the lowest level of Bulletin that should be
+shown in the User Interface. By default, the Bulletin level is set to WARN, which means it will display all warning and error-level
+bulletins.
+
+The right-hand side of the Settings tab contains an `Auto-terminate relationships' section. Each of the Relationships that is
+defined by the Processor is listed here, along with its description. In order for a Processor to be considered valid and
+able to run, each Relationship defined by the Processor must be either connected to a downstream component or auto-terminated.
+If a Relationship is auto-terminated, any FlowFile that is routed to that Relationship will be removed from the flow and
+its processing considered complete. Any Relationship that is already connected to a downstream component cannot be auto-terminated.
+The Relationship must first be removed from any Connection that uses it. Additionally, for any Relationship that is selected to be
+auto-terminated, the auto-termination status will be cleared (turned off) if the Relationship is added to a Connection.
+
+
+
+
+==== Scheduling Tab
+
+The second tab in the Processor Configuration dialog is the Scheduling Tab:
+
+image::scheduling-tab.png["Scheduling Tab"]
+
+The first configuration option is the Scheduling Strategy. There are three options for scheduling components:
+
+- *Timer driven*: This is the default mode. The Processor will be scheduled to run on a regular interval. The interval
+ at which the Processor is run is defined by the `Run schedule' option (see below).
+- *Event driven*: When this mode is selected, the Processor will be triggered to run by an event, and that event occurs when FlowFiles enter Connections
+ feeding this Processor. This mode is currently considered experimental and is not supported by all Processors. When this mode is
+ selected, the `Run schedule' option is not configurable, as the Processor is not triggered to run periodically but
+ as the result of an event. Additionally, this is the only mode for which the `Concurrent tasks'
+ option can be set to 0. In this case, the number of threads is limited only by the size of the Event-Driven Thread Pool that
+ the administrator has configured.
+- *CRON driven*: When using the CRON driven scheduling mode, the Processor is scheduled to run periodically, similar to the
+ Timer driven scheduling mode. However, the CRON driven mode provides significantly more flexibility at the expense of
+ increasing the complexity of the configuration. This value is made up of six fields, each separated by a space. These
+ fields include:
++
+ . Seconds
+ . Minutes
+ . Hours
+ . Day of Month
+ . Month
+ . Day of Week
+ . Year
++
+The value for each of these fields should be a number, range, or increment.
+Range here refers to a syntax of <number>-<number>.
+For example,the Seconds field could be set to 0-30, meaning that the Processor should only be scheduled if the time is 0 to 30 seconds
+after the minute. Additionally, a value of `*` indicates that all values are valid for this field. Multiple values can also
+be entered using a `,` as a separator: `0,5,10,15,30`.
+An increment is written as <start value>/<increment>. For example, settings a value of `0/10` for the seconds fields means that valid
+values are 0, 10, 20, 30, 40, and 50. However, if we change this to `5/10`, valid values become 5, 15, 25, 35, 45, and 55.
++
+For the Month field, valid values are 1 (January) through 12 (December).
++
+For the Day of Week field, valid values are 1 (Sunday) through 7 (Saturday). Additionally, a value of `L` may be appended to one of these
+values to indicate the last occurrence of this day in the month. For example, `1L` can be used to indicate the last Monday of the month.
+
+
+Next, the Scheduling Tab provides a configuration option named `Concurrent tasks.' This controls how many threads the Processor
+will use. Said a different way, this controls how many FlowFiles should be processed by this Processor at the same time. Increasing
+this value will typically allow the Processor to handle more data in the same amount of time. However, it does this by using system
+resources that then are not usable by other Processors. This essentially provides a relative weighting of Processors -- it controls
+how much of the system's resources should be allocated to this Processor instead of other Processors. This field is available for
+most Processors. There are, however, some types of Processors that can only be scheduled with a single Concurrent task.
+
+The ``Run schedule'' dictates how often the Processor should be scheduled to run. The valid values for this field depend on the selected
+Scheduling Strategy (see above). If using the Event driven Scheduling Strategy, this field is not available. When using the Timer driven
+Scheduling Strategy, this value is a time duration specified by a number followed by a time unit. For example, `1 second` or `5 mins`.
+The default value of `0 sec` means that the Processor should run as often as possible as long as it has data to process. This is true
+for any time duration of 0, regardless of the time unit (i.e., `0 sec`, `0 mins`, `0 days`). For an explanation of values that are
+applicable for the CRON driven Scheduling Strategy, see the description of the CRON driven Scheduling Strategy itself.
+
+The right-hand side of the tab contains a slider for choosing the `Run duration.' This controls how long the Processor should be scheduled
+to run each time that it is triggered. On the left-hand side of the slider, it is marked `Lower latency' while the right-hand side
+is marked `Higher throughput.' When a Processor finishes running, it must update the repository in order to transfer the FlowFiles to
+the next Connection. Updating the repository is expensive, so the more work that can be done at once before updating the repository,
+the more work the Processor can handle (Higher throughput). However, this means that the next Processor cannot start processing
+those FlowFiles until the previous Process updates this repository. As a result, the latency will be longer (the time required to process
+the FlowFile from beginning to end will be longer). As a result, the slider provides a spectrum from which the DFM can choose to favor
+Lower Latency or Higher Throughput.
+
+
+==== Properties Tab
+
+The Properties Tab provides a mechanism to configure Processor-specific behavior. There are no default properties. Each type of Processor
+must define which Properties make sense for its use case. Below, we see the Properties Tab for a RouteOnAttribute Processor:
+
+image::properties-tab.png["Properties Tab"]
+
+This Processor, by default, has only a single property: `Routing Strategy.' The default value is `Route on Property name.' Next to
+the name of this property is a small question-mark symbol (
+image:iconInfo.png["Question Mark"]
+). This help symbol is seen in other places throughout the User Interface, and it indicates that more information is available.
+Hovering over this symbol with the mouse will provide additional details about the property and the default value, as well as
+historical values that have been set for the Property.
+
+Clicking on the value for the property will allow a DFM to change the value. Depending on the values that are allowed for the property,
+the user is either provided a drop-down from which to choose a value or is given a text area to type a value:
+
+image::edit-property-dropdown.png["Edit Property with Dropdown"]
+
+In the top-right corner of the tab is a button for adding a New Property. Clicking this button will provide the DFM with a dialog to
+enter the name and value of a new property. Not all Processors allow User-Defined properties. In processors that do not allow them,
+the Processor becomes invalid when User-Defined properties are applied. RouteOnAttribute, however, does allow User-Defined properties.
+In fact, this Processor will not be valid until the user has added a property.
+
+image:edit-property-textarea.png["Edit Property with Text Area"]
+
+Note that after a User-Defined property has been added, an icon will appear on the right-hand side of that row (
+image:iconDelete.png["Delete Icon"]
+). Clicking this button will remove the User-Defined property from the Processor.
+
+Some processors also have an Advanced User Interface (UI) built into them. For example, the UpdateAttribute processor has an Advanced UI. To access the Advanced UI, click the `Advanced` button that appears at the bottom of the Configure Processor window. Only processors that have an Advanced UI will have this button.
+
+Some processors have properties that refer to other components, such as Controller Services, which also need to be configured. For example, the GetHTTP processor has an SSLContextService property, which refers to the StandardSSLContextService controller service. When DFMs want to configure this property but have not yet created and configured the controller service, they have the option to create the service on the spot, as depicted in the image below. For more information about configuring Controller Services, see the <<Controller_Services_and_Reporting_Tasks>> section.
+
+image:create-service-ssl-context.png["Create Service", width=700]
+
+==== Comments Tab
+
+The last tab in the Processor configuration dialog is the Comments tab. This tab simply provides an area for users to include
+whatever comments are appropriate for this component. Use of the Comments tab is optional:
+
+image::comments-tab.png["Comments Tab"]
+
+
+=== Additional Help
+
+The user may access additional documentation about each Processor's usage by right-clicking
+on the Processor and then selecting `Usage' from the context menu. Alternatively, clicking the `Help' link in the top-right
+corner of the User Interface will provide a Help page with all of the documentation, including usage documentation
+for all the Processors that are available. Clicking on the desired Processor in the list will display its usage documentation.
+
+
+
+[[Controller_Services_and_Reporting_Tasks]]
+=== Controller Services and Reporting Tasks
+
+While DFMs have the ability to create Controller Services from the Configure Processor window, there is also a central place within the User Interface for adding and configuring both Controller Services and Reporting Tasks. To get there, click on the Controller Settings button in the Management section of the toolbar.
+
+image:controller-settings-button.png["Controller Settings Button", width=200]
+
+The Controller Settings window has three tabs across the top: General, Controller Services, and Reporting Tasks. The General tab is for settings that pertain to general information about the NiFi instance. For example, here, the DFM can provide a unique name for the overall dataflow, as well as comments that describe the flow. Be aware that this information is visible to any other NiFi instance that connects remotely to this instance (using Remote Process Groups, a.k.a., Site-to-Site).
+
+The General tab also provides settings for the overall maximum thread counts of the instance, as well as the ability to click "Back-up flow" to create a backup copy of the current flow, which is saved by default in the /conf/archive directory.
+
+image:settings-general-tab.png["Controller Settings General Tab", width=700]
+
+To the right of the General tab is the Controller Services tab. From this tab, the DFM may click the "+" button in the upper-right corner to create a new Controller Service.
+
+image:controller-services-tab.png["Controller Services Tab", width=900]
+
+The Add Controller Service window opens. This window is similar to the Add Processor window. It provides a list of the available Controller Services on the right and a tag cloud, showing the most common catagory tags used for Controller Services, on the left. The DFM may click any tag in the tag cloud in order to narrow down the list of Controller Services to those that fit the categories desired. The DFM may also use the Filter field at the top of the window to search for the desired Contoller Service. Upon selecting a Controller Service from the list, the DFM can see a description of the the service below. Select the desired controller service and click Add, or simply double-click the name of the service to add it.
+
+image:add-controller-service-window.png["Add Controller Service Window", width=700]
+
+Once a Controller Service has been added, the DFM may configure it by clicking the Edit button (pencil icon) in the far-right column. Other buttons in this column include the Enable button (to enable a configured service), the Remove button, and the Usage button, which links to the documentation for the particular Controller Service.
+
+image:controller-services-edit-buttons.png["Controller Services Buttons"]
+
+When the DFM clicks the Edit button, a Configure Controller Service window opens. It has three tabs: Settings, Properties, and Comments. This window is similar to the Configure Processor window. The Settings tab provides a place for the DFM to give the Controller Service a unique name (if desired). It also lists the UUID for the service and provides a list of other components (processors or other controller services) that reference the service.
+
+image:configure-controller-service-settings.png["Configure Controller Service Settings", width=700]
+
+The Properties tab lists the various properties that apply to the particular controller service. As with configuring processors, the DFM may hover the mouse over the question mark icons to see more information about each property.
+
+image:configure-controller-service-properties.png["Configure Controller Service Properties", width=700]
+
+The Comments tab is just an open-text field, where the DFM may include comments about the service. After configuring a Controller Service, click the Apply button to apply the configuration and close the window, or click the Cancel button to cancel the changes and close the window.
+
+Note that after a Controller Service has been configured, it must be enabled in order to run. Do this using the Enable button in the far-right column of the Controller Services tab of the Controller Settings window. Then, in order to modify an existing/running controller service, the DFM needs to stop/disable it (as well as all referencing processors, reporting tasks, and controller services). Rather than having to hunt down each component that is referenced by that controller service, the DFM has the ability to stop/disable them when disabling the controller service in question. Likewise, when enabling a controller service, the DFM has the option to start/enable all referencing processors, reporting tasks, and controller services.
+
+The Reporting Tasks tab behaves similarly to the Controller Services tab. The DFM has the option to add Reporting Tasks and configure them in the same way as Controller Services.
+
+image:reporting-tasks-tab.png["Reporting Tasks Tab", width=900]
+
+Once a Reporting Task has been added, the DFM may configure it by clicking the Edit (pencil icon) in the far-right column. Other buttons in this column include the Start button, Remove button, and Usage button, which links to the documentation for the particular Reporting Task.
+
+image:reporting-tasks-edit-buttons2.png["Reporting Tasks Buttons"]
+
+When the DFM clicks the Edit button, a Configure Reporting Task window opens. It has three tabs: Settings, Properties, and Comments. This window is also similar to the Configure Processor window. The Settings tab provides a place for the DFM to give the Reporting Task a unique name (if desired). It also lists a UUID for the Reporting Task and provides settings for the task's Scheduling Strategy and Run Schedule (similar to the same settings in a processor). The DFM may hover the mouse over the question mark icons to see more information about each setting.
+
+image:configure-reporting-task-settings.png["Configure Reporting Task Settings", width=700]
+
+The Properties tab for a Reporting Task lists the properties that may be configured for the task. The DFM may hover the mouse over the question mark icons to see more information about each property.
+
+image:configure-reporting-task-properties.png["Configure Reporting Task Properties", width=700]
+
+The Comments tab is just an open-text field, where the DFM may include comments about the task. After configuring the Reporting Task, click the Apply button to apply the configuration and close the window, or click Cancel to cancel the changes and close the window.
+
+When you want to run the Reporting Task, click the Start button in the far-right column of the Reporting Tasks tab.
+
+
+
+=== Connecting Components
+
+Once processors and other components have been added to the graph and configured, the next step is to connect them
+to one another so that NiFi knows what to do with each FlowFile after it has been processed. This is accomplished by creating a
+Connection between each component. When the user hovers the mouse over the center of a component, a new Connection icon (
+image:addConnect.png["Connection Bubble"]
+) appears:
+
+image:processor-connection-bubble.png["Processor with Connection Bubble"]
+
+The user drags the Connection bubble from one component to another until the second component is highlighted. When the user
+releases the mouse, a `Create Connection' dialog appears. This dialog consists of two tabs: `Details' and `Settings'. They are
+discussed in detail below. Note that it is possible to draw a connection so that it loops back on the same processor. This can be
+useful if the DFM wants the processor to try to re-process FlowFiles if they go down a failure Relationship. To create this type of looping
+connection, simply drag the connection bubble away and then back to the same processor until it is highlighted. Then release the mouse and the same 'Create Connection' dialog appears.
+
+==== Details Tab
+
+The Details Tab of the 'Create Connection' dialog provides information about the source and destination components, including the component name, the
+component type, and the Process Group in which the component lives:
+
+image::create-connection.png["Create Connection"]
+
+Additionally, this tab provides the ability to choose which Relationships should be included in this Connection. At least one
+Relationship must be selected. If only one Relationship is available, it is automatically selected.
+
+*Note*: If multiple Connections are added with the same Relationship, any FlowFile that is routed to that Relationship will
+automatically be `cloned', and a copy will be sent to each of those Connections.
+
+==== Settings
+
+The Settings Tab provides the ability to configure the Connection's name, FlowFile expiration, Back Pressure thresholds, and
+Prioritization:
+
+image:connection-settings.png["Connection Settings"]
+
+The Connection name is optional. If not specified, the name shown for the Connection will be names of the Relationships
+that are active for the Connection.
+
+File expiration is a concept by which data that cannot be processed in a timely fashion can be automatically removed from the flow.
+This is useful, for example, when the volume of data is expected to exceed the volume that can be sent to a remote site.
+In this case, the expiration can be used in conjunction with Prioritizers to ensure that the highest priority data is
+processed first and then anything that cannot be processed within a certain time period (one hour, for example) can be dropped. The expiration period is based on the time that the data entered the NiFi instance. In other words, if the file expiration on a given connection is set to '1 hour', and a file that has been in the NiFi instance for one hour reaches that connection, it will expire. The default
+value of `0 sec` indicates that the data will never expire. When a file expiration other than '0 sec' is set, a small clock icon appears on the connection label, so the DFM can see it at-a-glance when looking at a flow on the graph.
+
+
+NiFi provides two configuration elements for Back Pressure. These thresholds indicate how much data should be
+allowed to exist in the queue before the component that is the source of the Connection is no longer scheduled to run.
+This allows the system to avoid being overrun with data. The first option provided is the ``Back pressure object threshold.''
+This is the number of FlowFiles that can be in the queue before back pressure is applied. The second configuration option
+is the ``Back pressure data size threshold.''
+This specifies the maximum amount of data (in size) that should be queued up before
+applying back pressure. This value is configured by entering a number followed by a data size (`B` for bytes, `KB` for
+kilobytes, `MB` for megabytes, `GB` for gigabytes, or `TB` for terabytes).
+
+The right-hand side of the tab provides the ability to prioritize the data in the queue so that higher priority data is
+processed first. Prioritizers can be dragged from the top (`Available prioritizers') to the bottom (`Selected prioritizers').
+Multiple prioritizers can be selected. The prioritizer that is at the top of the `Selected prioritizers' list is the highest
+priority. If two FlowFiles have the same value according to this prioritizer, the second prioritizer will determine which
+FlowFile to process first, and so on. If a prioritizer is no longer desired, it can then be dragged from the `Selected
+prioritizers' list to the `Available prioritizers' list.
+
+The following prioritizers are available:
+
+- *FirstInFirstOutPrioritizer*: Given two FlowFiles, the on that reached the connection first will be processed first.
+- *NewestFlowFileFirstPrioritizer*: Given two FlowFiles, the one that is newest in the dataflow will be processed first.
+- *OldestFlowFileFirstPrioritizer*: Given two FlowFiles, the on that is oldest in the dataflow will be processed first. This is the default scheme that is used if no prioritizers are selected.
+- *PriorityAttributePrioritizer*: Given two FlowFiles that both have a "priority" attribute, the one that has the highest priority value will be prprocessed first. Note that an UpdateAttribute processor should be used to add the "priority" attribute to the FlowFiles before they reach a connection that has this prioritizer set. Values for the "priority" attribute may be alphanumeric, where "a" is a higher priority than "z", and "1" is a higher priority than "9", for example.
+
+*Note*: After a connection has been drawn between two components, the connection's configuration may be changed, and the connection may be moved to a new destination; however, the processors on either side of the connection must be stopped before a configuration or destination change may be made.
+
+=== Processor Validation
+
+Before trying to start a Processor, it's important to make sure that the Processor's configuration is valid.
+A status indicator is shown in the top-left of the Processor. If the Processor is invalid, the indicator
+will show a yellow Warning indicator with an exclamation mark indicating that there is a problem:
+
+image::invalid-processor.png["Invalid Processor"]
+
+In this case, hovering over the indicator icon with the mouse will provide a tooltip showing all of the validation
+errors for the Processor. Once all of the validation errors have been addressed, the status indicator will change
+to a Stop icon, indicating that the Processor is valid and ready to be started but currently is not running:
+
+image::valid-processor.png["Valid Processor"]
+
+
+
+[[site-to-site]]
+=== Site-to-Site
+
+When sending data from one instance of NiFi to another, there are many different protocols that can be used. The preferred
+protocol, though, is the NiFi Site-to-Site Protocol. Site-to-Site makes it easy to transfer data from one NiFi instance to
+another easily, efficiently, and securely.
+
+Using Site-to-Site provides the following benefits:
+
+* Easy to configure
+** After entering the URL of the remote NiFi instance, the available ports (endpoints) are automatically discovered and provided in a drop-down list
+
+* Secure
+** Site-to-Site optionally makes use of Certificates in order to encrypt data and provide authentication and authorization. Each port can be configured
+ to allow only specific users, and only those users will be able to see that the port even exists. For information on configuring the Certificates,
+ see the
+link:administration-guide.html#security-configuration[Security Configuration] section of the
+link:administration-guide.html[Admin Guide].
+
+* Scalable
+** As nodes in the remote cluster change, those changes are automatically detected and data is scaled out across all nodes in the cluster.
+
+* Efficient
+** Site-to-Site allows batches of FlowFiles to be sent at once in order to avoid the overhead of establishing connections and making multiple
+ round-trip requests between peers.
+
+* Reliable
+** Checksums are automatically produced by both the sender and receiver and compared after the data has been transmitted, in order
+ to ensure that no corruption has occurred. If the checksums don't match, the transaction will simply be canceled and tried again.
+
+* Automatically load balanced
+** As nodes come online or drop out of the remote cluster, or a node's load becomes heavier or lighter, the amount of data that is directed
+ to that node will automatically be adjusted.
+
+* FlowFiles maintain attributes
+** When a FlowFile is transferred over this protocol, all of the FlowFile's attributes
+ are automatically transferred with it. This can be very advantageous in many situations, as all of the context and enrichment
+ that has been determined by one instance of NiFi travels with the data, making for easy routing of the data and allowing users
+ to easily inspect the data.
+
+* Adaptable
+** As new technologies and ideas emerge, the protocol for handling Site-to-Site communications are able to change with them. When a connection is
+ made to a remote NiFi instance, a handshake is performed in order to negotiate which protocol and which version of the protocol will be used.
+ This allows new capabilities to be added while still maintaining backward compatibility with all older instances. Additionally, if a vulnerability
+ or deficiency is ever discovered in a protocol, it allows a newer version of NiFi to forbid communication over the compromised versions of the protocol.
+
+In order to communicate with a remote NiFi instance via Site-to-Site, simply drag a <<remote_process_group,Remote Process Group>> onto the graph
+and enter the URL of the remote NiFi instance (for more information on the components of a Remote Process Group, see
+<<Remote_Group_Transmission,Remote Process Group Transmission>> section of this guide.) The URL is the same
+URL you would use to go to that instance's User Interface. At that point, you can drag a connection to or from the Remote Process Group
+in the same way you would drag a connection to or from a Processor or a local Process Group. When you drag the connection, you will have
+a chance to choose which Port to connect to. Note that it may take up to one minute for the Remote Process Group to determine
+which ports are available.
+
+If the connection is dragged starting from the Remote Process Group, the ports shown will be the Output Ports of the remote group,
+as this indicates that you will be pulling data from the remote instance. If the connection instead ends on the Remote Process Group,
+the ports shown will be the Input Ports of the remote group, as this implies that you will be pushing data to the remote instance.
+
+*Note*: if the remote instance is configured to use secure data transmission, you will see only ports that you are authorized to
+communicate with. For information on configuring NiFi to run securely, see the
+link:administration-guide.html[Admin Guide].
+
+In order to allow another NiFi instance to push data to your local instance, you can simply drag an <<input_port,Input Port>> onto the Root Process Group
+of your graph. After entering a name for the port, it will be added to your flow. You can now right-click on the Input Port and choose Configure in order
+to adjust the name and the number of concurrent tasks that are used for the port. If Site-to-Site is configured to run securely, you will also be given
+the ability to adjust who has access to the port. If secure, only those who have been granted access to communicate with the port will be able to see
+that the port exists.
+
+After being given access to a particular port, in order to see that port, the operator of a remote NiFi instance may need to right-click on their Remote
+Process Group and choose to "Refresh" the flow.
+
+Similar to an Input Port, a DataFlow Manager may choose to add an <<output_port,Output Port>> to the Root Process Group. The Output Port allows an
+authorized NiFi instance to remotely connect to your instance and pull data from the Output Port. Configuring the Output Port will again allow the
+DFM to control how many concurrent tasks are allowed, as well as which NiFi instances are authorized to pull data from the instance being configured.
+
+In addition to other instances of NiFi, some other applications may use a Site-to-Site client in order to push data to or receive data from a NiFi instance.
+For example, NiFi provides an Apache Storm spout and an Apache Spark Receiver that are able to pull data from NiFi's Root Group Output Ports.
+
+If your instance of NiFi is running securely, the first time that a client establishes a connection to your instance, the client will be forbidden and
+a request for an account for that client will automatically be generated. The client will need to be granted the 'NiFi' role in order to communicate
+via Site-to-Site. For more information on managing user accounts, see the
+link:administration-guide.html#controlling-levels-of-access[Controlling Levels of Access]
+section of the link:administration-guide.html[Admin Guide].
+
+For information on how to enable and configure Site-to-Site on a NiFi instance, see the
+link:administration-guide.html#site_to_site_properties[Site-to-Site Properties] section of the
+link:administration-guide.html[Admin Guide].
+
+
+
+
+=== Example Dataflow
+
+This section has described the steps required to build a dataflow. Now, to put it all together. The following example dataflow
+consists of just two processors: GenerateFlowFile and LogAttribute. These processors are normally used for testing, but they can also be used
+to build a quick flow for demonstration purposes and see NiFi in action.
+
+After you drag the GenerateFlowFile and LogAttribute processors to the graph and connect them (using the guidelines provided above), configure them as follows:
+
+* Generate FlowFile
+** On the Scheduling tab, set Run schedule to: 5 sec. Note that the GenerateFlowFile processor can create many FlowFiles very quickly; that's why setting the Run schedule is important so that this flow does not overwhelm the system NiFi is running on.
+** On the Properties tab, set File Size to: 10 kb
+
+* Log Attribute
+** On the Settings tab, under Auto-terminate relationships, select the checkbox next to Success. This will terminate FlowFiles after this processor has successfully processed them.
+** Also on the Settings tab, set the Bulletin level to Info. This way, when the dataflow is running, this processor will display the bulletin icon (see <<processor_anatomy>>), and the user may hover over it with the mouse to see the attributes that the processor is logging.
+
+The dataflow should look like the following:
+
+image::simple-flow.png["Simple Flow", width=900]
+
+
+Now see the following section on how to start and stop the dataflow. When the dataflow is running, be sure to note the statistical information that is displayed on the face of each processor (see <<processor_anatomy>>).
+
+
+
+== Command and Control of DataFlow
+
+When a component is added to the NiFi canvas, it is in the Stopped state. In order to cause the component to
+be triggered, the component must be started. Once started, the component can be stopped at any time. From a
+Stopped state, the component can be configured, started, or disabled.
+
+=== Starting a Component
+
+In order to start a component, the following conditions must be met:
+
+- The component's configuration must be valid.
+
+- All defined Relationships for the component must be connected to another component or auto-terminated.
+
+- The component must be stopped.
+
+- The component must be enabled.
+
+- The component must have no active tasks. For more information about active tasks, see the ``Anatomy of ...''
+sections under <<monitoring>> (<<processor_anatomy>>, <<process_group_anatomy>>, <<remote_group_anatomy>>).
+
+Components can be started by selecting all of the components to start and then clicking the Start icon (
+image:iconRun.png["Start"]
+) in the
+Actions Toolbar or by right-clicking a single component and choosing Start from the context menu.
+
+If starting a Process Group, all components within that Process Group (including child Process Groups) will
+be started, with the exception of those components that are invalid or disabled.
+
+Once started, the status indicator of a Processor will change to a Play symbol (
+image:iconRun.png["Run"]
+).
+
+
+=== Stopping a Component
+
+A component can be stopped any time that it is running. A component is stopped by right-clicking on the component
+and clicking Stop from the context menu, or by selecting the component and clicking the Stop icon (
+image:iconStop.png["Stop"]
+) in the Actions Toolbar.
+
+If a Process Group is stopped, all of the components within the Process Group (including child Process Groups)
+will be stopped.
+
+Once stopped, the status indicator of a component will change to the Stop symbol (
+image:iconStop.png["Stop"]
+).
+
+Stopping a component does not interrupt its currently running tasks. Rather, it stops scheduling new tasks to
+be performed. The number of active tasks is shown in the top-right corner of the Processor (see <<processor_anatomy>>
+for more information).
+
+=== Enabling/Disabling a Component
+
+When a component is enabled, it is able to be started. Users may choose to disable components when they are part of a
+dataflow that is still being assembled, for example. Typically, if a component is not intended to be run, the component
+is disabled, rather than being left in the Stopped state. This helps to distinguish between components that are
+intentionally not running and those that may have been stopped temporarily (for instance, to change the component's
+configuration) and inadvertently were never restarted.
+
+When it is desirable to re-enable a component, it can be enabled by selecting the component and
+clicking the Enable icon (
+image:iconEnable.png["Enable"]
+) in the Actions Toolbar. This is available only when the selected component or components are disabled.
+Alternatively, a component can be enabled by checking the checkbox next to the ``Enabled'' option in
+the Settings tab of the Processor configuration dialog or the configuration dialog for a Port.
+
+Once enabled, the component's status indicator will change to either Invalid (
+image:iconAlert.png["Invalid"]
+) or Stopped (
+image:iconStop.png["Stopped"]
+), depending on whether or not the component is valid.
+
+A component is then disabled by selecting the component and clicking the Disable icon (
+image:iconDisable.png["Disable"]
+) in the Actions Toolbar, or by clearing the checkbox next to the ``Enabled'' option in the Settings tab
+of the Processor configuration dialog or the configuration dialog for a Port.
+
+Only Ports and Processors can be enabled and disabled.
+
+
+[[Remote_Group_Transmission]]
+=== Remote Process Group Transmission
+
+Remote Process Groups provide a mechanism for sending data to or retrieving data from a remote instance
+of NiFi. When a Remote Process Group (RPG) is added to the canvas, it is added with the Transmision Disabled,
+as indicated by the icon (
+image:iconTransmissionInactive.png["Transmission Disabled"]
+) in the top-left corner. When Transmission is Disabled, it can be enabled by right-clicking on the
+RPG and clicking the ``Enable Transmission'' menu item. This will cause all ports for which there is a Connection
+to begin transmitting data. This will cause the status indicator to then change to the Transmission Enabled icon (
+image:iconTransmissionActive.png["Transmission Enabled"]
+).
+
+If there are problems communicating with the Remote Process Group, a Warning indicator (
+image:iconAlert.png["Warning"]
+) may instead be present in the top-left corner. Hovering over this Warning indicator with the mouse will provide
+more information about the problem.
+
+[[Remote_Port_Configuration]]
+==== Individual Port Transmission
+
+There are times when the DFM may want to either enable or disable transmission for only a specific
+Port within the Remote Process Group. This can be accomplished by right-clicking on the Remote Process Group
+and choosing the ``Remote ports'' menu item. This provides a configuration dialog from which each Port can be
+configured:
+
+image::remote-group-ports-dialog.png["Remote Process Groups"]
+
+The left-hand side lists all of the Input Ports that the remote instance of NiFi allows data to be sent to.
+The right-hand side lists all of the Output Ports from which this instance is able to pull data.
+If the remote instance is using secure communications (the URL of the NiFi instance begins with `https://`,
+rather than `http://`), any Ports that the remote instance has not made available to this instance will not
+be shown.
+
+*Note*: If a Port that is expected to be shown is not shown in this dialog, ensure that the instance has proper
+permissions and that the Remote Process Group's flow is current. This can be checked by closing the Port
+Configuration Dialog and looking at the bottom-right corner of the Remote Process Group. The date at which
+the flow was last refreshed is shown. If the flow appears to be outdated, it can be updated by right-clicking
+on the Remote Process Group and selecting ``Refresh flow.'' (See <<remote_group_anatomy>> for more information).
+
+Each Port is shown with the Port name, followed by its description, currently configured number of Concurrent
+tasks, and whether or not data sent to this port will be compressed. To the left of this information is a switch
+to turn the Port on or off. Those Ports that have no Connections attached to them are grayed out:
+
+image::remote-port-connection-status.png["Remote Port Statuses"]
+
+The on/off switch provides a mechanism to enable and disable transmission for each Port in the Remote
+Process Group independently. Those Ports that are connected but are not currently transmitting can be
+configured by clicking the pencil icon (
+image:iconEdit.png["Edit"]
+) below the on/off switch. Clicking this icon will allow the DFM to change the number of Concurrent tasks and whether
+or not compression should be used when transmitting data to or from this Port.
+
+
+
+[[navigating]]
+== Navigating within a DataFlow
+
+NiFi provides various mechanisms for getting around a dataflow. The <<User_Interface>> section discussed various ways to navigate around
+the NiFi graph; however, once a flow exists on the graph, there are additional ways to get from one component to another. The <<User Interface>> section showed that when multiple Process Groups exist in a flow, breadcrumbs appear under the toolbar, providing a way to navigate between them. In addition, to enter a Process Group that is currently visible on the graph, simply double-click it, thereby "drilling down" into it. Connections also provide a way to jump from one location to another within the flow. Right-click on a connection and select "Go to source" or "Go to destination" in order to jump to one end of the connection or another. This can be very useful in large, complex dataflows, where the connection lines may be long and span large areas of the graph. Finally, all components provide the ability to jump forward or backward within the flow. Right-click any component (e.g., a processor, process group, port, etc.) and select either "Upstream connections" or "Downstream connec
tions". A dialog window will open, showing the available upstream or downstream connections that the user may jump to. This can be especially useful when trying to follow a dataflow in a backward direction. It is typically easy to follow the path of a dataflow from start to finish, drilling down into nested process groups; however, it can be more difficult to follow the dataflow in the other direction.
+
+
+
+[[monitoring]]
+== Monitoring of DataFlow
+
+NiFi provides a great deal of information about the status of the DataFlow in order to monitor the
+health and status. The Status bar provides information about the overall system health
+(See <<status_bar>> above for more information). Processors, Process Groups, and Remote Process Groups
+provide fine-grained details about their operations. Connections and Process Groups provide information
+about the amount of data in their queues. The Summary Page provides information about all of the components
+on the graph in a tabular format and also provides System Diagnostics information that includes disk usage,
+CPU utilization, and Java Heap and Garbage Collection information. In a clustered environment, this
+information is available per-node or as aggregates across the entire cluster. We will explore each of these
+monitoring artifacts below.
+
+
+[[processor_anatomy]]
+=== Anatomy of a Processor
+
+NiFi provides a significant amount of information about each Processor on the canvas. The following diagram
+shows the anatomy of a Processor:
+
+image:processor-anatomy.png["Anatomy of a Processor"]
+
+The image outlines the following elements:
+
+- *Processor Type*: NiFi provides several different types of Processors in order to allow for a wide range
+ of tasks to be performed. Each type of Processor is designed to perform one specific task. The Processor
+ type (PutFile, in this example) describes the task that this Processor performs. In this case, the
+ Processor writes a FlowFile to disk - or ``Puts'' a FlowFile to a File.
+
+- *Bulletin Indicator*: When a Processor logs that some event has occurred, it generates a Bulletin to notify
+ those who are monitoring NiFi via the User Interface. The DFM is able to configure which
+ bulletins should be displayed in the User Interface by updating the ``Bulletin level'' field in the
+ ``Settings'' tab of the Processor configuration dialog. The default value is `WARN`, which means that only
+ warnings and errors will be displayed in the UI. This icon is not present unless a Bulletin exists for this
+ Processor. When it is present, hovering over the icon with the mouse will provide a tooltip explaining the
+ message provided by the Processor as well as the Bulletin level. If the instance of NiFi is clustered,
+ it will also show the Node that emitted the Bulletin. Bulletins automatically expire after five minutes.
+
+- *Status Indicator*: Shows the current Status of the Processor. The following indicators are possible:
+ ** image:iconRun.png["Running"]
+ *Running*: The Processor is currently running.
+ ** image:iconStop.png["Stopped"]
+ *Stopped*: The Processor is valid and enabled but is not running.
+ ** image:iconAlert.png["Invalid"]
+ *Invalid*: The Processor is enabled but is not currently valid and cannot be started.
+ Hovering over this icon will provide a tooltip indicating why the Processor is not valid.
+ ** image:iconDisable.png["Disabled"]
+ *Disabled*: The Processor is not running and cannot be started until it has been enabled.
+ This status does not indicate whether or not the Processor is valid.
+
+- *Processor Name*: This is the user-defined name of the Processor. By default, the name of the Processor is
+ the same as the Processor Type. In the example, this value is "Copy to /review".
+
+- *Active Tasks*: The number of tasks that this Processor is currently executing. This number is constrained
+ by the ``Concurrent tasks'' setting in the ``Scheduling'' tab of the Processor configuration dialog.
+ Here, we can see that the Processor is currently performing two tasks. If the NiFi instance is clustered,
+ this value represents the number of tasks that are currently executing across all nodes in the cluster.
+
+- *5-Minute Statistics*: The Processor shows several different statistics in tabular form. Each of these
+ statistics represents the amount of work that has been performed in the past five minutes. If the NiFi
+ instance is clustered, these values indicate how much work has been done by all of the Nodes combined
+ in the past five minutes. These metrics are:
+
+ ** *In*: The amount of data that the Processor has pulled from the queues of its incoming Connections.
+ This value is represented as <count> / <size> where <count> is the number of FlowFiles that have been
+ pulled from the queues and <size> is the total size of those FlowFiles' content. In this example,
+ the Processor has pulled 884 FlowFiles from the input queues, for a total of 8.85 megabytes (MB).
+ ** *Read/Write*: The total size of the FlowFile content that the Processor has read from disk and written
+ to disk. This provides valuable information about the I/O performance that this Processor requires.
+ Some Processors may only read the data without writing anything while some will not read the data but
+ will only write data. Others will neither read nor write data, and some Processors will both read
+ and write data. In this example, we see that in the past five minutes, this Processor has read 4.7
+ MB of the FlowFile content and has written 4.7 MB as well. This is what we would expect,
+ since this Processor simply copies the contents of a FlowFile to disk. Note, however, that this is
+ not the same as the amount of data that it pulled from its input queues. This is because some of
+ the files that it pulled from the input queues already exist in the output directory, and the
+ Processor is configured to route FlowFiles to failure when this occurs. Therefore, for those files
+ which already existed in the output directory, data was neither read nor written to disk.
+ ** *Out*: The amount of data that the Processor has transferred to its outbound Connections. This does
+ not include FlowFiles that the Processor removes itself, or FlowFiles that are routed to connections
+ that are auto-terminated. Like the ``In'' metric above, this value is represented as <count> / <size>
+ where <count> is the number of FlowFiles that have been transferred to outbound Connections and <size>
+ is the total size of those FlowFiles' content. In this example, all of the Relationships are configured to be
+ auto-terminated, so no FlowFiles are reported as having been transferred Out.
+ ** *Tasks/Time*: The number of times that this Processor has been triggered to run in the past 5 minutes, and
+ the amount of time taken to perform those tasks. The format of the time is <hour>:<minute>:<second>. Note
+ that the amount of time taken can exceed five minutes, because many tasks can be executed in parallel. For
+ instance, if the Processor is scheduled to run with 60 Concurrent tasks, and each of those tasks takes one
+ second to complete, it is possible that all 60 tasks will be completed in a single second. However, in this
+ case we will see the Time metric showing that it took 60 seconds, instead of 1 second. This time can be
+ thought of as ``System Time,'' or said another way, this value is 60 seconds because that's the amount of
+ time it would have taken to perform the action if only a single concurrent task were used.
+
+
+
+
+
+[[process_group_anatomy]]
+=== Anatomy of a Process Group
+
+The Process Group provides a mechanism for grouping components together into a logical construct in order
+to organize the DataFlow in a way that makes it more understandable from a higher level.
+The following image highlights the different elements that make up the anatomy of a Process Group:
+
+image::process-group-anatomy.png["Anatomy of a Process Group"]
+
+The Process Group consists of the following elements:
+
+- *Name*: This is the user-defined name of the Process Group. This name is set when the Process Group
+ is added to the canvas. The name can later by changed by right-clicking on the Process Group and clicking
+ the ``Configure'' menu option. In this example, the name of the Process Group is ``Process Group ABC.''
+
+- *Bulletin Indicator*: When a child component of a Process Group emits a bulletin, that bulletin is propagated to
+ the component's parent Process Group, as well. When any component has an active Bulletin, this indicator will appear,
+ allowing the user to hover over the icon with the mouse to see Bulletin.
+
+- *Active Tasks*: The number of tasks that are currently executing by the components within this
+ Process Group. Here, we can see that the Process Group is currently performing one task. If the
+ NiFi instance is clustered, this value represents the number of tasks that are currently executing
+ across all nodes in the cluster.
+
+- *Comments*: When the Process Group is added to the canvas, the user is given the option of specifying Comments in order
+ to provide information about the Process Group. The comments can later be changed by right-clicking on the Process
+ Group and clicking the ``Configure'' menu option. In this example, the Comments are set to ``Example Process Group.''
+
+- *Statistics*: Process Groups provide statistics about the amount of data that has been processed by the Process Group in
+ the past 5 minutes as well as the amount of data currently enqueued within the Process Group. The following elements
+ comprise the ``Statistics'' portion of a Process Group:
+ ** *Queued*: The number of FlowFiles currently enqueued within the Process Group.
+ This field is represented as <count> / <size> where <count> is the number of FlowFiles that are
+ currently enqueued in the Process Group and <size> is the total size of those FlowFiles' content. In this example,
+ the Process Group currently has 1,738 FlowFiles enqueued; those FlowFiles have a total size of 350.03 megabytes (MB).
+
+ ** *In*: The number of FlowFiles that have been transferred into the Process Group through all of its Input Ports
+ over the past 5 minutes. This field is represented as <count> / <size> where <count> is the number of FlowFiles that
+ have entered the Process Group in the past 5 minutes and <size> is the total size of those FlowFiles' content.
+ In this example, 686 FlowFiles have entered the Process Group and their total size is 214.01 MB.
+
+ ** *Read/Write*: The total size of the FlowFile content that the components within the Process Group have
+ read from disk and written to disk. This provides valuable information about the I/O performance that this
+ Process Group requires. In this example, we see that in the past five minutes, components within this
+ Process Group have read 72.9 MB of the FlowFile content and have written 686.65 MB.
+
+ ** *Out*: The number of FlowFiles that have been transferred out of the Process Group through its Output Ports
+ over the past 5 minutes. This field is represented as <count> / <size> where <count> is the number of FlowFiles that
+ have exited the Process Group in the past 5 minutes and <size> is the total size of those FlowFiles' content.
+ In this example, 657 FlowFiles have exited the Process Group and their total size is 477.74 MB.
+
+- *Component Counts*: The Component Counts element provides information about how many components of each type exist
+ within the Process Group. The following provides information about each of these icons and their meanings:
+
+ ** image:iconInputPortSmall.png["Input Port", width=16]
+ *Input Ports*: The number of Input Ports that exist directly within this Process Group. This does not include any
+ Input Ports that exist within child Process Groups, as child groups' ports cannot be accessed directly.
+
+ ** image:iconOutputPortSmall.png["Output Port"]
+ *Output Ports*: The number of Output Ports that exist directly within this Process Group. This does not include any
+ Output Ports that exist within child Process Group as child groups' ports cannot be accessed directly.
+
+ ** image:iconTransmissionActive.png["Transmission Active"]
+ *Transmitting Ports*: The number of Remote Process Group Ports that currently are configured to transmit data to remote
+ instances of NiFi or pull data from remote instances of NiFi.
+
+ ** image:iconTransmissionInactive.png["Transmission Inactive"]
+ *Non-Transmitting Ports*: The number of Remote Process Group Ports that are currently connected to components within this
+ Process Group but currently have their transmission disabled.
+
+ ** image:iconRun.png["Running"]
+ *Running Components*: The number of Processors, Input Ports, and Output Ports that are currently running within this
+ Process Group.
+
+ ** image:iconStop.png["Stopped Components"]
+ *Stopped Components*: The number of Processors, Input Ports, and Output Ports that are currently not running but are
+ valid and enabled. These components are ready to be started.
+
+ ** image:iconAlert.png["Invalid Components"]
+ *Invalid Components*: The number of Processors, Input Ports, and Output Ports that are enabled but are currently
+ not in a valid state. This may be due to misconfigured properties or missing Relationships.
+
+ ** image:iconDisable.png["Disabled Components"]
+ *Disabled Components*: The number of Processors, Input Ports, and Output Ports that are currently disabled. These
+ components may or may not be valid. If the Process Group is started, these components will not cause any errors
+ but will not be started.
+
+
+
+
+
+[[remote_group_anatomy]]
+=== Anatomy of a Remote Process Group
+
+When creating a DataFlow, it is often necessary to transfer data from one instance of NiFi to another.
+In this case, the remote instance of NiFi can be thought of as a Process Group. For this reason, NiFi
+provides the concept of a Remote Process Group. From the User Interface, the Remote Process Group
+looks similar to the Process Group. However, rather than showing information about the inner workings
+and state of a Remote Process Group, such as queue sizes, the information rendered about a Remote
+Process Group is related to the interaction that occurs between this instance of NiFi and the remote
+instance.
+
+image::remote-group-anatomy.png["Anatomy of a Remote Process Group"]
+
+The image above shows the different elements that make up a Remote Process Group. Here, we provide an
+explanation of the icons and details about the information provided.
+
+- *Transmission Status*: The Transmission Status indicates whether or not data Transmission between this
+ instance of NiFi and the remote instance is currently enabled. The icon shown will be the
+ Transmission Enabled icon (
+image:iconTransmissionActive.png["Transmission Active"]
+ ) if any of the Input Ports or Output Ports is currently configured to transmit or the Transmission
+ Disabled icon (
+image:iconTransmissionInactive.png["Transmission Inactive"]
+ ) if all of the Input Ports and Output Ports that are currently connected are stopped.
+
+- *Remote Instance Name*: This is the name of the NiFi instance that was reported by the remote instance.
+ When the Remote Process Group is first created, before this information has been obtained, the URL
+ of the remote instance will be shown here instead.
+
+- *Remote Instance URL*: This is the URL of the remote instance that the Remote Process Group points to.
+ This URL is entered when the Remote Process Group is added to the canvas and it cannot be changed.
+
+- *Secure Indicator*: This icon indicates whether or not communications with the remote NiFi instance are
+ secure. If communications with the remote instance are secure, this will be indicated by the ``locked''
+ icon (
+image:iconSecure.png["Secure"]
+ ). If the communications are not secure, this will be indicated by the ``unlocked'' icon (
+image:iconNotSecure.png["Not Secure"]
+ ). If the communications are secure, this instance of NiFi will not be able to communicate with the
+ remote instance until an administrator for the remote instance grants access. Whenever the Remote Process
+ Group is added to the canvas, this will automatically initiate a request to have a user for this instance of NiFi created on the
+ remote instance. This instance will be unable to communicate with the remote instance until an administrator
+ on the remote instance adds the user to the system and adds the ``NiFi'' role to the user.
+ In the event that communications are not secure, the Remote Process Group is able to receive data from anyone,
+ and the data is not encrypted while it is transferred between instances of NiFi.
+
+- *Input Ports*: This section shows three pieces of information:
+ ** image:iconInputPortSmall.png["Input Ports"]
+ *Input Ports*: The number of Input Ports that are available to send data to on the remote instance of NiFi.
+ If the remote instance is secure, only the ports to which this instance of NiFi has been granted access
+ will be counted.
+
+ ** image:iconTransmissionActive.png["Transmitting"]
+ *Transmitting Ports*: The number of Input Ports to which this NiFi is connected and currently configured to
+ send data to. Ports can be turned on and off by enabling and disabling transmission on the Remote Process
+ Group (see <<Remote_Group_Transmission>>) or via the <<Remote_Port_Configuration>> dialog.
+
+ ** image:iconTransmissionInactive.png["Not Transmitting"]
+ *Non-Transmitting Ports*: The number of Input Ports to which this NiFi is connected but is not currently configured
+ to send data to. Ports can be turned on and off by enabling and disabling transmission on the Remote Process
+ Group (see <<Remote_Group_Transmission>>) or via the <<Remote_Port_Configuration>> dialog.
+
+- *Output Ports*: Similar to the ``Input Ports'' section above, this element shows three pieces of information:
+ ** image:iconOutputPortSmall.png["Output Ports"]
+ *Output Ports*: The number of Output Ports that are available to pull data from the remote instance of NiFi.
+ If the remote instance is secure, only the ports to which this instance of NiFi has been granted access
+ will be counted.
+
+ ** image:iconTransmissionActive.png["Transmitting"]
+ *Transmitting Ports*: The number of Output Ports from whcih this NiFi is connected and currently configured
+ to pull data from. Ports can be turned on and off by enabling and disabling transmission on the Remote Process
+ Group (see <<Remote_Group_Transmission>>) or via the <<Remote_Port_Configuration>> dialog.
+
+ ** image:iconTransmissionInactive.png["Not Transmitting"]
+ *Non-Transmitting Ports*: The number of Output Ports to which this NiFi is connected but is not currently configured
+ to pull data from. Ports can be turned on and off by enabling and disabling transmission on the Remote Process
+ Group (see <<Remote_Group_Transmission>>) or via the <<Remote_Port_Configuration>> dialog.
+
+- *5-Minute Statistics*: Two statistics are shown for Remote Process Groups: *Sent* and *Received*. Both of these are
+ in the format <count> / <size> where <count> is the number of FlowFiles that have been sent or received in the previous
+ five minutes and <size> is the total size of those FlowFiles' content.
+
+- *Comments*: The Comments that are provided for a Remote Process Group are not comments added by the users of this NiFi but
+ rather the Comments added by the administrators of the remote instance. These comments indicate the purpose of the NiFi
+ instance as a whole.
+
+- *Last Refreshed Time*: The information that is pulled from a remote instance and rendered on the Remote Process Group
+ in the User Interface is periodically refreshed in the background. This element indicates the time at which that refresh
+ last happened, or if the information has not been refreshed for a significant amount of time, the value will change to
+ indicate _Remote flow not current_. NiFi can be triggered to initiate a refresh of this information by right-clicking
+ on the Remote Process Group and choosing the ``Refresh flow'' menu item.
+
+
+
+
+[[Summary_Page]]
+=== Summary Page
+
+While the NiFi canvas is useful for understanding how the configured DataFlow is laid out, this view is not always optimal
+when trying to discern the status of the system. In order to help the user understand how the DataFlow is functioning
+at a higher level, NiFi provides a Summary page. This page is available in the Management Toolbar in the top-right corner
+of the User Interface. See the <<User_Interface>> section for more information about the location of this toolbar.
+
+The Summary Page is opened by clicking the Summary icon (
+image:iconSummary.png["Summary"]
+) from the Management Toolbar. This opens the Summary table dialog:
+
+image::summary-table.png["Summary Table"]
+
+This dialog provides a great deal of information about each of the components on the graph. Below, we have annotated
+the different elements within the dialog in order to make the discussion of the dialog easier.
+
+image::summary-annotated.png["Summary Table Annotated"]
+
+The Summary page consists mostly of a table that provides information about each of the components on the canvas. Above this
+table is a set of five tabs that can be used to view the different types of components. The information provided in the table
+is the same information that is provided for each component on the canvas. Each of the columns in the table may be sorted by
+double-clicking on the heading of the column. For more on the types of information displayed, see the sections
+<<processor_anatomy>>, <<process_group_anatomy>>, and <<remote_group_anatomy>> above.
+
+The Summary page also includes the following elements:
+
+- *Bulletin Indicator*: As in other places throughout the User Interface, when this icon is present, hovering over the icon will
+ provide information about the Bulletin that was generated, including the message, the severity level, the time at which
+ the Bulletin was generated, and (in a clustered environment) the node that generated the Bulletin. Like all the columns in the
+ Summary table, this column where bulletins are shown may be sorted
+ by double-clicking on the heading so that all the currently existing bulletins are shown at the top of the list.
+
+- *Details*: Clicking the Details icon will provide the user with the details of the component. This dialog is the same as the
+ dialog provided when the user right-clicks on the component and chooses the ``View configuration'' menu item.
+
+- *Go To*: Clicking this button will close the Summary page and take the user directly to the component on the NiFi canvas. This
+ may change the Process Group that the user is currently in. This icon is not available if the Summary page has been opened
+ in a new browser tab or window (by clicking the ``Pop Out'' button, as described below).
+
+- *Stats History*: Clicking the Stats History icon will open a new dialog that shows a historical view of the statistics that
+ are rendered for this component. See the section <<Stats_History>> for more information.
+
+- *Refresh*: The Refresh button allows the user to refresh the information displayed without closing the dialog and opening it
+ again. The time at which the information was last refreshed is shown just to the right of the Refresh button. The information
+ on the page is not automatically refreshed.
+
+- *Filter*: The Filter element allows users to filter the contents of the Summary table by typing in all or part of some criteria,
+ such as a Processor Type or Processor Name. The types of filters available differ according to the selected tab. For instance,
+ if viewing the Processor tab, the user is able to filter by name or by type. When viewing the Connections tab, the user
+ is able to filter by source name, destination name, or Connection name. The filter is automatically applied when the contents
+ of the text box are changed. Below the text box is an indicator of how many entries in the table match the filter and how many
+ entries exist in the table.
+
+- *Pop-Out*: When monitoring a flow, it is helpful to be able to open the Summary table in a separate browser tab or window. The
+ Pop-Out button, next to the Close button, will cause the entire Summary dialog to be opened in a new browser tab or window
+ (depending on the configuration of the browser). Once the page is ``popped out'', the dialog is closed in the original
+ browser tab/window. In the new tab/window, the Pop-Out button and the Go-To button will no longer be available.
+
+- *System Diagnostics*: The System Diagnostics window provides information about how the system is performing with respect to
+ system resource utilization. While this is intended mostly for administrators, it is provided in this view because it
+ does provide a summary of the system. This dialog shows information such as CPU utilization, how full the disks are,
+ and Java-specific metrics, such as memory size and utilization, as well as Garbage Collection information.
+
+
+
+
+[[Stats_History]]
+=== Historical Statistics of a Component
+
+While the Summary table and the canvas show numeric statistics pertaining to the performance of a component over the
+past five minutes, it is often useful to have a view of historical statistics as well. This information is available
+by right-clicking on a component and choosing the ``Stats'' menu option or by clicking on the Stats History in the Summary page (see <<Summary_Page>>
+for more information).
+
+The amount of historical information that is sto
<TRUNCATED>
[13/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-docs/src/main/assembly/dependencies.xml
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/assembly/dependencies.xml b/nifi-docs/src/main/assembly/dependencies.xml
new file mode 100644
index 0000000..6f6279b
--- /dev/null
+++ b/nifi-docs/src/main/assembly/dependencies.xml
@@ -0,0 +1,44 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<assembly>
+ <id>resources</id>
+ <formats>
+ <format>zip</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${project.build.directory}/generated-docs/</directory>
+ <outputDirectory>/html/</outputDirectory>
+ </fileSet>
+ </fileSets>
+ <files>
+ <file>
+ <source>./LICENSE</source>
+ <outputDirectory>./</outputDirectory>
+ <destName>LICENSE</destName>
+ <fileMode>0644</fileMode>
+ <filtered>true</filtered>
+ </file>
+ <file>
+ <source>./NOTICE</source>
+ <outputDirectory>./</outputDirectory>
+ <destName>NOTICE</destName>
+ <fileMode>0644</fileMode>
+ <filtered>true</filtered>
+ </file>
+ </files>
+</assembly>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-external/README.md
----------------------------------------------------------------------
diff --git a/nifi-external/README.md b/nifi-external/README.md
new file mode 100644
index 0000000..649ad17
--- /dev/null
+++ b/nifi-external/README.md
@@ -0,0 +1,19 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+# nifi-external
+
+The nifi-external module is a location where components can be developed by the NiFi team
+that are not intended to be used directly by NiFi but are to be used within other frameworks
+in order to integrate with NiFi.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-external/nifi-example-bundle/nifi-nifi-example-nar/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-external/nifi-example-bundle/nifi-nifi-example-nar/pom.xml b/nifi-external/nifi-example-bundle/nifi-nifi-example-nar/pom.xml
new file mode 100644
index 0000000..6ca950b
--- /dev/null
+++ b/nifi-external/nifi-example-bundle/nifi-nifi-example-nar/pom.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-example-bundle</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-example-nar</artifactId>
+ <packaging>nar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-nifi-example-processors</artifactId>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-external/nifi-example-bundle/nifi-nifi-example-processors/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-external/nifi-example-bundle/nifi-nifi-example-processors/pom.xml b/nifi-external/nifi-example-bundle/nifi-nifi-example-processors/pom.xml
new file mode 100644
index 0000000..5bdbb7c
--- /dev/null
+++ b/nifi-external/nifi-example-bundle/nifi-nifi-example-processors/pom.xml
@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-example-bundle</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-nifi-example-processors</artifactId>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-processor-utils</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-mock</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-simple</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>src/main/resources/file.txt</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-external/nifi-example-bundle/nifi-nifi-example-processors/src/main/java/org/apache/nifi/processors/WriteResourceToStream.java
----------------------------------------------------------------------
diff --git a/nifi-external/nifi-example-bundle/nifi-nifi-example-processors/src/main/java/org/apache/nifi/processors/WriteResourceToStream.java b/nifi-external/nifi-example-bundle/nifi-nifi-example-processors/src/main/java/org/apache/nifi/processors/WriteResourceToStream.java
new file mode 100644
index 0000000..5d595b4
--- /dev/null
+++ b/nifi-external/nifi-example-bundle/nifi-nifi-example-processors/src/main/java/org/apache/nifi/processors/WriteResourceToStream.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.io.OutputStreamCallback;
+
+@Tags({ "example", "resources" })
+@CapabilityDescription("This example processor loads a resource from the nar and writes it to the FlowFile content")
+public class WriteResourceToStream extends AbstractProcessor {
+
+ public static final Relationship REL_SUCCESS = new Relationship.Builder()
+ .name("success")
+ .description("files that were successfully processed").build();
+ public static final Relationship REL_FAILURE = new Relationship.Builder()
+ .name("failure")
+ .description("files that were not successfully processed").build();
+
+ private Set<Relationship> relationships;
+
+ private String resourceData;
+
+ @Override
+ protected void init(final ProcessorInitializationContext context) {
+
+ final Set<Relationship> relationships = new HashSet<Relationship>();
+ relationships.add(REL_SUCCESS);
+ relationships.add(REL_FAILURE);
+ this.relationships = Collections.unmodifiableSet(relationships);
+
+ try {
+ this.resourceData = IOUtils.toString(Thread.currentThread()
+ .getContextClassLoader().getResourceAsStream("file.txt"));
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to load resources", e);
+ }
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return this.relationships;
+ }
+
+ @OnScheduled
+ public void onScheduled(final ProcessContext context) {
+
+ }
+
+ @Override
+ public void onTrigger(final ProcessContext context,
+ final ProcessSession session) throws ProcessException {
+ FlowFile flowFile = session.get();
+ if (flowFile == null) {
+ return;
+ }
+
+ try {
+ flowFile = session.write(flowFile, new OutputStreamCallback() {
+
+ @Override
+ public void process(OutputStream out) throws IOException {
+ IOUtils.write(resourceData, out);
+
+ }
+ });
+ session.transfer(flowFile, REL_SUCCESS);
+ } catch (ProcessException ex) {
+ getLogger().error("Unable to process", ex);
+ session.transfer(flowFile, REL_FAILURE);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-external/nifi-example-bundle/nifi-nifi-example-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
----------------------------------------------------------------------
diff --git a/nifi-external/nifi-example-bundle/nifi-nifi-example-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-external/nifi-example-bundle/nifi-nifi-example-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
new file mode 100644
index 0000000..9448541
--- /dev/null
+++ b/nifi-external/nifi-example-bundle/nifi-nifi-example-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
@@ -0,0 +1,15 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+org.apache.nifi.processors.WriteResourceToStream
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-external/nifi-example-bundle/nifi-nifi-example-processors/src/main/resources/file.txt
----------------------------------------------------------------------
diff --git a/nifi-external/nifi-example-bundle/nifi-nifi-example-processors/src/main/resources/file.txt b/nifi-external/nifi-example-bundle/nifi-nifi-example-processors/src/main/resources/file.txt
new file mode 100644
index 0000000..f0e1d6c
--- /dev/null
+++ b/nifi-external/nifi-example-bundle/nifi-nifi-example-processors/src/main/resources/file.txt
@@ -0,0 +1 @@
+this came from a resource
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-external/nifi-example-bundle/nifi-nifi-example-processors/src/test/java/org/apache/nifi/processors/WriteResourceToStreamTest.java
----------------------------------------------------------------------
diff --git a/nifi-external/nifi-example-bundle/nifi-nifi-example-processors/src/test/java/org/apache/nifi/processors/WriteResourceToStreamTest.java b/nifi-external/nifi-example-bundle/nifi-nifi-example-processors/src/test/java/org/apache/nifi/processors/WriteResourceToStreamTest.java
new file mode 100644
index 0000000..7a84df5
--- /dev/null
+++ b/nifi-external/nifi-example-bundle/nifi-nifi-example-processors/src/test/java/org/apache/nifi/processors/WriteResourceToStreamTest.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors;
+
+import org.junit.Assert;
+
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.Before;
+import org.junit.Test;
+
+public class WriteResourceToStreamTest {
+
+ private TestRunner testRunner;
+
+ @Before
+ public void init() {
+ testRunner = TestRunners.newTestRunner(WriteResourceToStream.class);
+ }
+
+ @Test
+ public void testProcessor() {
+ testRunner.enqueue(new byte[] { 1, 2, 3, 4, 5 });
+ testRunner.run();
+ testRunner.assertAllFlowFilesTransferred(WriteResourceToStream.REL_SUCCESS, 1);
+ final byte[] data = testRunner
+ .getFlowFilesForRelationship(WriteResourceToStream.REL_SUCCESS).get(0)
+ .toByteArray();
+ final String stringData = new String(data);
+ Assert.assertEquals("this came from a resource", stringData);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-external/nifi-example-bundle/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-external/nifi-example-bundle/pom.xml b/nifi-external/nifi-example-bundle/pom.xml
new file mode 100644
index 0000000..c62f093
--- /dev/null
+++ b/nifi-external/nifi-example-bundle/pom.xml
@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-external</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-example-bundle</artifactId>
+ <packaging>pom</packaging>
+
+ <modules>
+ <module>nifi-nifi-example-processors</module>
+ <module>nifi-nifi-example-nar</module>
+ </modules>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-nifi-example-processors</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-external/nifi-spark-receiver/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-external/nifi-spark-receiver/pom.xml b/nifi-external/nifi-spark-receiver/pom.xml
new file mode 100644
index 0000000..e290713
--- /dev/null
+++ b/nifi-external/nifi-spark-receiver/pom.xml
@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-external</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-spark-receiver</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-streaming_2.10</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-site-to-site-client</artifactId>
+ </dependency>
+ </dependencies>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-external/nifi-spark-receiver/src/main/java/org/apache/nifi/spark/NiFiDataPacket.java
----------------------------------------------------------------------
diff --git a/nifi-external/nifi-spark-receiver/src/main/java/org/apache/nifi/spark/NiFiDataPacket.java b/nifi-external/nifi-spark-receiver/src/main/java/org/apache/nifi/spark/NiFiDataPacket.java
new file mode 100644
index 0000000..484c2a9
--- /dev/null
+++ b/nifi-external/nifi-spark-receiver/src/main/java/org/apache/nifi/spark/NiFiDataPacket.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.spark;
+
+import java.util.Map;
+
+/**
+ * <p>
+ * The NiFiDataPacket provides a packaging around a NiFi FlowFile. It wraps both
+ * a FlowFile's content and its attributes so that they can be processed by
+ * Spark
+ * </p>
+ */
+public interface NiFiDataPacket {
+
+ /**
+ * @return the contents of a NiFi FlowFile
+ */
+ byte[] getContent();
+
+ /**
+ * @return a Map of attributes that are associated with the NiFi FlowFile
+ */
+ Map<String, String> getAttributes();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-external/nifi-spark-receiver/src/main/java/org/apache/nifi/spark/NiFiReceiver.java
----------------------------------------------------------------------
diff --git a/nifi-external/nifi-spark-receiver/src/main/java/org/apache/nifi/spark/NiFiReceiver.java b/nifi-external/nifi-spark-receiver/src/main/java/org/apache/nifi/spark/NiFiReceiver.java
new file mode 100644
index 0000000..8cbf60c
--- /dev/null
+++ b/nifi-external/nifi-spark-receiver/src/main/java/org/apache/nifi/spark/NiFiReceiver.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.spark;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.nifi.remote.Transaction;
+import org.apache.nifi.remote.TransferDirection;
+import org.apache.nifi.remote.client.SiteToSiteClient;
+import org.apache.nifi.remote.client.SiteToSiteClientConfig;
+import org.apache.nifi.remote.protocol.DataPacket;
+import org.apache.nifi.stream.io.StreamUtils;
+import org.apache.spark.storage.StorageLevel;
+import org.apache.spark.streaming.receiver.Receiver;
+
+/**
+ * <p>
+ * The <code>NiFiReceiver</code> is a Reliable Receiver that provides a way to
+ * pull data from Apache NiFi so that it can be processed by Spark Streaming.
+ * The NiFi Receiver connects to NiFi instance provided in the config and
+ * requests data from the OutputPort that is named. In NiFi, when an OutputPort
+ * is added to the root process group, it acts as a queue of data for remote
+ * clients. This receiver is then able to pull that data from NiFi reliably.
+ * </p>
+ *
+ * <p>
+ * It is important to note that if pulling data from a NiFi cluster, the URL
+ * that should be used is that of the NiFi Cluster Manager. The Receiver will
+ * automatically handle determining the nodes in that cluster and pull from
+ * those nodes as appropriate.
+ * </p>
+ *
+ * <p>
+ * In order to use the NiFiReceiver, you will need to first build a
+ * {@link SiteToSiteClientConfig} to provide to the constructor. This can be
+ * achieved by using the {@link SiteToSiteClient.Builder}. Below is an example
+ * snippet of driver code to pull data from NiFi that is running on
+ * localhost:8080. This example assumes that NiFi exposes and OutputPort on the
+ * root group named "Data For Spark". Additionally, it assumes that the data
+ * that it will receive from this OutputPort is text data, as it will map the
+ * byte array received from NiFi to a UTF-8 Encoded string.
+ * </p>
+ *
+ * <code>
+ * <pre>
+ * {@code
+ * Pattern SPACE = Pattern.compile(" ");
+ *
+ * // Build a Site-to-site client config
+ * SiteToSiteClientConfig config = new SiteToSiteClient.Builder()
+ * .setUrl("http://localhost:8080/nifi")
+ * .setPortName("Data For Spark")
+ * .buildConfig();
+ *
+ * SparkConf sparkConf = new SparkConf().setAppName("NiFi-Spark Streaming example");
+ * JavaStreamingContext ssc = new JavaStreamingContext(sparkConf, new Duration(1000L));
+ *
+ * // Create a JavaReceiverInputDStream using a NiFi receiver so that we can pull data from
+ * // specified Port
+ * JavaReceiverInputDStream<NiFiDataPacket> packetStream =
+ * ssc.receiverStream(new NiFiReceiver(clientConfig, StorageLevel.MEMORY_ONLY()));
+ *
+ * // Map the data from NiFi to text, ignoring the attributes
+ * JavaDStream<String> text = packetStream.map(new Function<NiFiDataPacket, String>() {
+ * public String call(final NiFiDataPacket dataPacket) throws Exception {
+ * return new String(dataPacket.getContent(), StandardCharsets.UTF_8);
+ * }
+ * });
+ *
+ * // Split the words by spaces
+ * JavaDStream<String> words = text.flatMap(new FlatMapFunction<String, String>() {
+ * public Iterable<String> call(final String text) throws Exception {
+ * return Arrays.asList(SPACE.split(text));
+ * }
+ * });
+ *
+ * // Map each word to the number 1, then aggregate by key
+ * JavaPairDStream<String, Integer> wordCounts = words.mapToPair(
+ * new PairFunction<String, String, Integer>() {
+ * public Tuple2<String, Integer> call(String s) {
+ * return new Tuple2<String, Integer>(s, 1);
+ * }
+ * }).reduceByKey(new Function2<Integer, Integer, Integer>() {
+ * public Integer call(Integer i1, Integer i2) {
+ * return i1 + i2;
+ * }
+ * }
+ * );
+ *
+ * // print the results
+ * wordCounts.print();
+ * ssc.start();
+ * ssc.awaitTermination();
+ * }
+ * </pre>
+ * </code>
+ */
+public class NiFiReceiver extends Receiver<NiFiDataPacket> {
+
+ private static final long serialVersionUID = 3067274587595578836L;
+ private final SiteToSiteClientConfig clientConfig;
+
+ public NiFiReceiver(final SiteToSiteClientConfig clientConfig, final StorageLevel storageLevel) {
+ super(storageLevel);
+ this.clientConfig = clientConfig;
+ }
+
+ @Override
+ public void onStart() {
+ final Thread thread = new Thread(new ReceiveRunnable());
+ thread.setDaemon(true);
+ thread.setName("NiFi Receiver");
+ thread.start();
+ }
+
+ @Override
+ public void onStop() {
+ }
+
+ class ReceiveRunnable implements Runnable {
+
+ public ReceiveRunnable() {
+ }
+
+ @Override
+ public void run() {
+ try {
+ final SiteToSiteClient client = new SiteToSiteClient.Builder().fromConfig(clientConfig).build();
+ try {
+ while (!isStopped()) {
+ final Transaction transaction = client.createTransaction(TransferDirection.RECEIVE);
+ DataPacket dataPacket = transaction.receive();
+ if (dataPacket == null) {
+ transaction.confirm();
+ transaction.complete();
+
+ // no data available. Wait a bit and try again
+ try {
+ Thread.sleep(1000L);
+ } catch (InterruptedException e) {
+ }
+
+ continue;
+ }
+
+ final List<NiFiDataPacket> dataPackets = new ArrayList<>();
+ do {
+ // Read the data into a byte array and wrap it along with the attributes
+ // into a NiFiDataPacket.
+ final InputStream inStream = dataPacket.getData();
+ final byte[] data = new byte[(int) dataPacket.getSize()];
+ StreamUtils.fillBuffer(inStream, data);
+
+ final Map<String, String> attributes = dataPacket.getAttributes();
+ final NiFiDataPacket NiFiDataPacket = new NiFiDataPacket() {
+ @Override
+ public byte[] getContent() {
+ return data;
+ }
+
+ @Override
+ public Map<String, String> getAttributes() {
+ return attributes;
+ }
+ };
+
+ dataPackets.add(NiFiDataPacket);
+ dataPacket = transaction.receive();
+ } while (dataPacket != null);
+
+ // Confirm transaction to verify the data
+ transaction.confirm();
+
+ store(dataPackets.iterator());
+
+ transaction.complete();
+ }
+ } finally {
+ try {
+ client.close();
+ } catch (final IOException ioe) {
+ reportError("Failed to close client", ioe);
+ }
+ }
+ } catch (final IOException ioe) {
+ restart("Failed to receive data from NiFi", ioe);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-external/nifi-storm-spout/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-external/nifi-storm-spout/pom.xml b/nifi-external/nifi-storm-spout/pom.xml
new file mode 100644
index 0000000..1691cce
--- /dev/null
+++ b/nifi-external/nifi-storm-spout/pom.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-external</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-storm-spout</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-core</artifactId>
+ <version>0.9.5</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-site-to-site-client</artifactId>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-external/nifi-storm-spout/src/main/java/org/apache/nifi/storm/NiFiDataPacket.java
----------------------------------------------------------------------
diff --git a/nifi-external/nifi-storm-spout/src/main/java/org/apache/nifi/storm/NiFiDataPacket.java b/nifi-external/nifi-storm-spout/src/main/java/org/apache/nifi/storm/NiFiDataPacket.java
new file mode 100644
index 0000000..4c399e0
--- /dev/null
+++ b/nifi-external/nifi-storm-spout/src/main/java/org/apache/nifi/storm/NiFiDataPacket.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.storm;
+
+import java.util.Map;
+
+/**
+ * <p>
+ * The NiFiDataPacket provides a packaging around a NiFi FlowFile. It wraps both
+ * a FlowFile's content and its attributes so that they can be processed by
+ * Storm
+ * </p>
+ */
+public interface NiFiDataPacket {
+
+ /**
+ * @return the contents of a NiFi FlowFile
+ */
+ byte[] getContent();
+
+ /**
+ * @return a Map of attributes that are associated with the NiFi FlowFile
+ */
+ Map<String, String> getAttributes();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-external/nifi-storm-spout/src/main/java/org/apache/nifi/storm/NiFiSpout.java
----------------------------------------------------------------------
diff --git a/nifi-external/nifi-storm-spout/src/main/java/org/apache/nifi/storm/NiFiSpout.java b/nifi-external/nifi-storm-spout/src/main/java/org/apache/nifi/storm/NiFiSpout.java
new file mode 100644
index 0000000..64dac6f
--- /dev/null
+++ b/nifi-external/nifi-storm-spout/src/main/java/org/apache/nifi/storm/NiFiSpout.java
@@ -0,0 +1,266 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.storm;
+
+import backtype.storm.spout.SpoutOutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.topology.base.BaseRichSpout;
+import backtype.storm.tuple.Fields;
+import backtype.storm.tuple.Values;
+import backtype.storm.utils.Utils;
+import org.apache.nifi.remote.Transaction;
+import org.apache.nifi.remote.TransferDirection;
+import org.apache.nifi.remote.client.SiteToSiteClient;
+import org.apache.nifi.remote.client.SiteToSiteClientConfig;
+import org.apache.nifi.remote.protocol.DataPacket;
+import org.apache.nifi.stream.io.StreamUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.LinkedBlockingQueue;
+
+/**
+ * <p>
+ * The <code>NiFiSpout</code> provides a way to pull data from Apache NiFi so
+ * that it can be processed by Apache Storm. The NiFi Spout connects to a NiFi
+ * instance provided in the config and requests data from the OutputPort that
+ * is named. In NiFi, when an OutputPort is added to the root process group,
+ * it acts as a queue of data for remote clients. This spout is then able to
+ * pull that data from NiFi reliably.
+ * </p>
+ *
+ * <p>
+ * It is important to note that if pulling data from a NiFi cluster, the URL
+ * that should be used is that of the NiFi Cluster Manager. The Receiver will
+ * automatically handle determining the nodes in that cluster and pull from
+ * those nodes as appropriate.
+ * </p>
+ *
+ * <p>
+ * In order to use the NiFiSpout, you will need to first build a
+ * {@link SiteToSiteClientConfig} to provide to the constructor. This can be
+ * achieved by using the {@link SiteToSiteClient.Builder}. Below is an example
+ * snippet of driver code to pull data from NiFi that is running on
+ * localhost:8080. This example assumes that NiFi exposes an OutputPort on the
+ * root group named "Data For Storm". Additionally, it assumes that the data
+ * that it will receive from this OutputPort is text data, as it will map the
+ * byte array received from NiFi to a UTF-8 Encoded string.
+ * </p>
+ *
+ * <code>
+ * <pre>
+ * {@code
+ *
+ * // Build a Site-To-Site client config
+ * SiteToSiteClientConfig clientConfig = new SiteToSiteClient.Builder()
+ * .url("http://localhost:8080/nifi")
+ * .portName("Data for Storm")
+ * .buildConfig();
+ *
+ * // Build a topology starting with a NiFiSpout
+ * TopologyBuilder builder = new TopologyBuilder();
+ * builder.setSpout("nifi", new NiFiSpout(clientConfig));
+ *
+ * // Add a bolt that prints the attributes and content
+ * builder.setBolt("print", new BaseBasicBolt() {
+ * @Override
+ * public void execute(Tuple tuple, BasicOutputCollector collector) {
+ * NiFiDataPacket dp = (NiFiDataPacket) tuple.getValueByField("nifiDataPacket");
+ * System.out.println("Attributes: " + dp.getAttributes());
+ * System.out.println("Content: " + new String(dp.getContent()));
+ * }
+ *
+ * @Override
+ * public void declareOutputFields(OutputFieldsDeclarer declarer) {}
+ *
+ * }).shuffleGrouping("nifi");
+ *
+ * // Submit the topology running in local mode
+ * Config conf = new Config();
+ * LocalCluster cluster = new LocalCluster();
+ * cluster.submitTopology("test", conf, builder.createTopology());
+ *
+ * Utils.sleep(90000);
+ * cluster.shutdown();
+ * }
+ * </pre>
+ * </code>
+ */
+public class NiFiSpout extends BaseRichSpout {
+
+ private static final long serialVersionUID = 3067274587595578836L;
+
+ public static final Logger LOGGER = LoggerFactory.getLogger(NiFiSpout.class);
+
+ public static final String NIFI_DATA_PACKET = "nifiDataPacket";
+
+ private NiFiSpoutReceiver spoutReceiver;
+ private LinkedBlockingQueue<NiFiDataPacket> queue;
+ private SpoutOutputCollector spoutOutputCollector;
+
+ private final SiteToSiteClientConfig clientConfig;
+ private final List<String> attributeNames;
+
+ /**
+ * @param clientConfig
+ * configuration used to build the SiteToSiteClient
+ */
+ public NiFiSpout(SiteToSiteClientConfig clientConfig) {
+ this(clientConfig, null);
+ }
+
+ /**
+ *
+ * @param clientConfig
+ * configuration used to build the SiteToSiteClient
+ * @param attributeNames
+ * names of FlowFile attributes to be added as values to each tuple, in addition
+ * to the nifiDataPacket value on all tuples
+ *
+ */
+ public NiFiSpout(SiteToSiteClientConfig clientConfig, List<String> attributeNames) {
+ this.clientConfig = clientConfig;
+ this.attributeNames = (attributeNames == null ? new ArrayList<String>() : attributeNames);
+ }
+
+ @Override
+ public void open(Map map, TopologyContext topologyContext, SpoutOutputCollector spoutOutputCollector) {
+ this.spoutOutputCollector = spoutOutputCollector;
+ this.queue = new LinkedBlockingQueue<>(1000);
+
+ this.spoutReceiver = new NiFiSpoutReceiver();
+ this.spoutReceiver.setDaemon(true);
+ this.spoutReceiver.setName("NiFi Spout Receiver");
+ this.spoutReceiver.start();
+ }
+
+ @Override
+ public void nextTuple() {
+ NiFiDataPacket data = queue.poll();
+ if (data == null) {
+ Utils.sleep(50);
+ } else {
+ // always start with the data packet
+ Values values = new Values(data);
+
+ // add additional values based on the specified attribute names
+ for (String attributeName : attributeNames) {
+ if (data.getAttributes().containsKey(attributeName)) {
+ values.add(data.getAttributes().get(attributeName));
+ }
+ }
+
+ spoutOutputCollector.emit(values);
+ }
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
+ final List<String> fieldNames = new ArrayList<>();
+ fieldNames.add(NIFI_DATA_PACKET);
+ fieldNames.addAll(attributeNames);
+ outputFieldsDeclarer.declare(new Fields(fieldNames));
+ }
+
+ @Override
+ public void close() {
+ super.close();
+ spoutReceiver.shutdown();
+ }
+
+ class NiFiSpoutReceiver extends Thread {
+
+ private boolean shutdown = false;
+
+ public synchronized void shutdown() {
+ this.shutdown = true;
+ }
+
+ @Override
+ public void run() {
+ try {
+ final SiteToSiteClient client = new SiteToSiteClient.Builder().fromConfig(clientConfig).build();
+ try {
+ while (!shutdown) {
+ final Transaction transaction = client.createTransaction(TransferDirection.RECEIVE);
+ DataPacket dataPacket = transaction.receive();
+ if (dataPacket == null) {
+ transaction.confirm();
+ transaction.complete();
+
+ // no data available. Wait a bit and try again
+ try {
+ Thread.sleep(1000L);
+ } catch (InterruptedException e) {
+ }
+
+ continue;
+ }
+
+ final List<NiFiDataPacket> dataPackets = new ArrayList<>();
+ do {
+ // Read the data into a byte array and wrap it along with the attributes
+ // into a NiFiDataPacket.
+ final InputStream inStream = dataPacket.getData();
+ final byte[] data = new byte[(int) dataPacket.getSize()];
+ StreamUtils.fillBuffer(inStream, data);
+
+ final Map<String, String> attributes = dataPacket.getAttributes();
+ final NiFiDataPacket niFiDataPacket = new NiFiDataPacket() {
+ @Override
+ public byte[] getContent() {
+ return data;
+ }
+
+ @Override
+ public Map<String, String> getAttributes() {
+ return attributes;
+ }
+ };
+
+ dataPackets.add(niFiDataPacket);
+ dataPacket = transaction.receive();
+ } while (dataPacket != null);
+
+ // Confirm transaction to verify the data
+ transaction.confirm();
+
+ for (NiFiDataPacket dp : dataPackets) {
+ queue.offer(dp);
+ }
+
+ transaction.complete();
+ }
+ } finally {
+ try {
+ client.close();
+ } catch (final IOException ioe) {
+ LOGGER.error("Failed to close client", ioe);
+ }
+ }
+ } catch (final IOException ioe) {
+ LOGGER.error("Failed to receive data from NiFi", ioe);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-external/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-external/pom.xml b/nifi-external/pom.xml
new file mode 100644
index 0000000..40aebef
--- /dev/null
+++ b/nifi-external/pom.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-external</artifactId>
+ <packaging>pom</packaging>
+ <modules>
+ <module>nifi-spark-receiver</module>
+ <module>nifi-storm-spout</module>
+ <module>nifi-example-bundle</module>
+ </modules>
+</project>
+
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-maven-archetypes/nifi-processor-bundle-archetype/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-maven-archetypes/nifi-processor-bundle-archetype/pom.xml b/nifi-maven-archetypes/nifi-processor-bundle-archetype/pom.xml
new file mode 100644
index 0000000..8df532b
--- /dev/null
+++ b/nifi-maven-archetypes/nifi-processor-bundle-archetype/pom.xml
@@ -0,0 +1,47 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-maven-archetypes</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-processor-bundle-archetype</artifactId>
+ <packaging>maven-archetype</packaging>
+
+ <build>
+ <extensions>
+ <extension>
+ <groupId>org.apache.maven.archetype</groupId>
+ <artifactId>archetype-packaging</artifactId>
+ <version>2.2</version>
+ </extension>
+ </extensions>
+
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-archetype-plugin</artifactId>
+ <version>2.2</version>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-maven-archetypes/nifi-processor-bundle-archetype/src/main/resources/META-INF/maven/archetype-metadata.xml
----------------------------------------------------------------------
diff --git a/nifi-maven-archetypes/nifi-processor-bundle-archetype/src/main/resources/META-INF/maven/archetype-metadata.xml b/nifi-maven-archetypes/nifi-processor-bundle-archetype/src/main/resources/META-INF/maven/archetype-metadata.xml
new file mode 100644
index 0000000..3dc2ba8
--- /dev/null
+++ b/nifi-maven-archetypes/nifi-processor-bundle-archetype/src/main/resources/META-INF/maven/archetype-metadata.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<archetype-descriptor name="${artifactId}">
+ <requiredProperties>
+ <requiredProperty key="artifactBaseName" />
+ <requiredProperty key="nifiVersion" >
+ <defaultValue>0.2.1-SNAPSHOT</defaultValue>
+ </requiredProperty>
+ <requiredProperty key="package">
+ <defaultValue>${groupId}.processors.${artifactBaseName}</defaultValue>
+ </requiredProperty>
+ </requiredProperties>
+ <fileSets>
+ <fileSet filtered="true" packaged="false" encoding="UTF-8">
+ <directory>nifi-__artifactBaseName__-processors</directory>
+ <excludes>
+ <exclude>**/*.java</exclude>
+ </excludes>
+ </fileSet>
+ <fileSet filtered="true" packaged="true" encoding="UTF-8">
+ <directory>nifi-__artifactBaseName__-processors/src/main/java</directory>
+ </fileSet>
+ <fileSet filtered="true" packaged="true" encoding="UTF-8">
+ <directory>nifi-__artifactBaseName__-processors/src/test/java</directory>
+ </fileSet>
+ <fileSet filtered="true" packaged="false" encoding="UTF-8">
+ <directory>nifi-__artifactBaseName__-nar</directory>
+ </fileSet>
+ </fileSets>
+</archetype-descriptor>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-maven-archetypes/nifi-processor-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__-nar/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-maven-archetypes/nifi-processor-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__-nar/pom.xml b/nifi-maven-archetypes/nifi-processor-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__-nar/pom.xml
new file mode 100644
index 0000000..60875a6
--- /dev/null
+++ b/nifi-maven-archetypes/nifi-processor-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__-nar/pom.xml
@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>${groupId}</groupId>
+ <artifactId>${rootArtifactId}</artifactId>
+ <version>${version}</version>
+ </parent>
+
+ <artifactId>nifi-${artifactBaseName}-nar</artifactId>
+ <version>${version}</version>
+ <packaging>nar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>${groupId}</groupId>
+ <artifactId>nifi-${artifactBaseName}-processors</artifactId>
+ <version>${version}</version>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-maven-archetypes/nifi-processor-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__-processors/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-maven-archetypes/nifi-processor-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__-processors/pom.xml b/nifi-maven-archetypes/nifi-processor-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__-processors/pom.xml
new file mode 100644
index 0000000..5cd42ee
--- /dev/null
+++ b/nifi-maven-archetypes/nifi-processor-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__-processors/pom.xml
@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>${groupId}</groupId>
+ <artifactId>${rootArtifactId}</artifactId>
+ <version>${version}</version>
+ </parent>
+
+ <artifactId>nifi-${artifactBaseName}-processors</artifactId>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-processor-utils</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-mock</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-simple</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.11</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-maven-archetypes/nifi-processor-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__-processors/src/main/java/MyProcessor.java
----------------------------------------------------------------------
diff --git a/nifi-maven-archetypes/nifi-processor-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__-processors/src/main/java/MyProcessor.java b/nifi-maven-archetypes/nifi-processor-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__-processors/src/main/java/MyProcessor.java
new file mode 100644
index 0000000..7b70dca
--- /dev/null
+++ b/nifi-maven-archetypes/nifi-processor-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__-processors/src/main/java/MyProcessor.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ${package};
+
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.PropertyValue;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.*;
+import org.apache.nifi.annotation.behavior.ReadsAttribute;
+import org.apache.nifi.annotation.behavior.ReadsAttributes;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.SeeAlso;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+
+import java.util.*;
+
+@Tags({"example"})
+@CapabilityDescription("Provide a description")
+@SeeAlso({})
+@ReadsAttributes({@ReadsAttribute(attribute="", description="")})
+@WritesAttributes({@WritesAttribute(attribute="", description="")})
+public class MyProcessor extends AbstractProcessor {
+
+ public static final PropertyDescriptor MY_PROPERTY = new PropertyDescriptor
+ .Builder().name("My Property")
+ .description("Example Property")
+ .required(true)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+
+ public static final Relationship MY_RELATIONSHIP = new Relationship.Builder()
+ .name("my_relationship")
+ .description("Example relationship")
+ .build();
+
+ private List<PropertyDescriptor> descriptors;
+
+ private Set<Relationship> relationships;
+
+ @Override
+ protected void init(final ProcessorInitializationContext context) {
+ final List<PropertyDescriptor> descriptors = new ArrayList<PropertyDescriptor>();
+ descriptors.add(MY_PROPERTY);
+ this.descriptors = Collections.unmodifiableList(descriptors);
+
+ final Set<Relationship> relationships = new HashSet<Relationship>();
+ relationships.add(MY_RELATIONSHIP);
+ this.relationships = Collections.unmodifiableSet(relationships);
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return this.relationships;
+ }
+
+ @Override
+ public final List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return descriptors;
+ }
+
+ @OnScheduled
+ public void onScheduled(final ProcessContext context) {
+
+ }
+
+ @Override
+ public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
+ FlowFile flowFile = session.get();
+ if ( flowFile == null ) {
+ return;
+ }
+
+ // TODO implement
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-maven-archetypes/nifi-processor-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
----------------------------------------------------------------------
diff --git a/nifi-maven-archetypes/nifi-processor-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-maven-archetypes/nifi-processor-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
new file mode 100644
index 0000000..0454176
--- /dev/null
+++ b/nifi-maven-archetypes/nifi-processor-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
@@ -0,0 +1,15 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+${package}.MyProcessor
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-maven-archetypes/nifi-processor-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__-processors/src/test/java/MyProcessorTest.java
----------------------------------------------------------------------
diff --git a/nifi-maven-archetypes/nifi-processor-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__-processors/src/test/java/MyProcessorTest.java b/nifi-maven-archetypes/nifi-processor-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__-processors/src/test/java/MyProcessorTest.java
new file mode 100644
index 0000000..2b709f8
--- /dev/null
+++ b/nifi-maven-archetypes/nifi-processor-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__-processors/src/test/java/MyProcessorTest.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ${package};
+
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.Before;
+import org.junit.Test;
+
+
+public class MyProcessorTest {
+
+ private TestRunner testRunner;
+
+ @Before
+ public void init() {
+ testRunner = TestRunners.newTestRunner(MyProcessor.class);
+ }
+
+ @Test
+ public void testProcessor() {
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-maven-archetypes/nifi-processor-bundle-archetype/src/main/resources/archetype-resources/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-maven-archetypes/nifi-processor-bundle-archetype/src/main/resources/archetype-resources/pom.xml b/nifi-maven-archetypes/nifi-processor-bundle-archetype/src/main/resources/archetype-resources/pom.xml
new file mode 100644
index 0000000..f1677e5
--- /dev/null
+++ b/nifi-maven-archetypes/nifi-processor-bundle-archetype/src/main/resources/archetype-resources/pom.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-nar-bundles</artifactId>
+ <version>${nifiVersion}</version>
+ </parent>
+
+ <groupId>${groupId}</groupId>
+ <artifactId>${artifactId}</artifactId>
+ <version>${version}</version>
+ <packaging>pom</packaging>
+
+ <modules>
+ <module>nifi-${artifactBaseName}-processors</module>
+ <module>nifi-${artifactBaseName}-nar</module>
+ </modules>
+
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-maven-archetypes/nifi-service-bundle-archetype/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-maven-archetypes/nifi-service-bundle-archetype/pom.xml b/nifi-maven-archetypes/nifi-service-bundle-archetype/pom.xml
new file mode 100644
index 0000000..a18d0b3
--- /dev/null
+++ b/nifi-maven-archetypes/nifi-service-bundle-archetype/pom.xml
@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-maven-archetypes</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-service-bundle-archetype</artifactId>
+ <packaging>maven-archetype</packaging>
+
+ <build>
+ <extensions>
+ <extension>
+ <groupId>org.apache.maven.archetype</groupId>
+ <artifactId>archetype-packaging</artifactId>
+ <version>2.2</version>
+ </extension>
+ </extensions>
+
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-archetype-plugin</artifactId>
+ <version>2.2</version>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/META-INF/maven/archetype-metadata.xml
----------------------------------------------------------------------
diff --git a/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/META-INF/maven/archetype-metadata.xml b/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/META-INF/maven/archetype-metadata.xml
new file mode 100644
index 0000000..5ed0b1d
--- /dev/null
+++ b/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/META-INF/maven/archetype-metadata.xml
@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<archetype-descriptor name="${artifactId}">
+ <requiredProperties>
+ <requiredProperty key="artifactBaseName" />
+ <requiredProperty key="nifiVersion" >
+ <defaultValue>0.3.0-SNAPSHOT</defaultValue>
+ </requiredProperty>
+ <requiredProperty key="package">
+ <defaultValue>${groupId}.${artifactBaseName}</defaultValue>
+ </requiredProperty>
+ </requiredProperties>
+ <fileSets>
+ <fileSet filtered="true" packaged="false" encoding="UTF-8">
+ <directory>nifi-__artifactBaseName__-api</directory>
+ <excludes>
+ <exclude>**/*.java</exclude>
+ </excludes>
+ </fileSet>
+ <fileSet filtered="true" packaged="true" encoding="UTF-8">
+ <directory>nifi-__artifactBaseName__-api/src/main/java</directory>
+ </fileSet>
+ <fileSet filtered="true" packaged="false" encoding="UTF-8">
+ <directory>nifi-__artifactBaseName__</directory>
+ <excludes>
+ <exclude>**/*.java</exclude>
+ </excludes>
+ </fileSet>
+ <fileSet filtered="true" packaged="true" encoding="UTF-8">
+ <directory>nifi-__artifactBaseName__/src/main/java</directory>
+ </fileSet>
+ <fileSet filtered="true" packaged="true" encoding="UTF-8">
+ <directory>nifi-__artifactBaseName__/src/test/java</directory>
+ </fileSet>
+ <fileSet filtered="true" packaged="false" encoding="UTF-8">
+ <directory>nifi-__artifactBaseName__-api-nar</directory>
+ </fileSet>
+ <fileSet filtered="true" packaged="false" encoding="UTF-8">
+ <directory>nifi-__artifactBaseName__-nar</directory>
+ </fileSet>
+ </fileSets>
+</archetype-descriptor>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__-api-nar/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__-api-nar/pom.xml b/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__-api-nar/pom.xml
new file mode 100644
index 0000000..52a3264
--- /dev/null
+++ b/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__-api-nar/pom.xml
@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>${groupId}</groupId>
+ <artifactId>${rootArtifactId}</artifactId>
+ <version>${version}</version>
+ </parent>
+
+ <artifactId>nifi-${artifactBaseName}-api-nar</artifactId>
+ <version>${version}</version>
+ <packaging>nar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-standard-services-api-nar</artifactId>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>${groupId}</groupId>
+ <artifactId>nifi-${artifactBaseName}-api</artifactId>
+ <version>${version}</version>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__-api/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__-api/pom.xml b/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__-api/pom.xml
new file mode 100644
index 0000000..26c11d6
--- /dev/null
+++ b/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__-api/pom.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>${groupId}</groupId>
+ <artifactId>${rootArtifactId}</artifactId>
+ <version>${version}</version>
+ </parent>
+
+ <artifactId>nifi-${artifactBaseName}-api</artifactId>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__-api/src/main/java/MyService.java
----------------------------------------------------------------------
diff --git a/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__-api/src/main/java/MyService.java b/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__-api/src/main/java/MyService.java
new file mode 100644
index 0000000..df58f5e
--- /dev/null
+++ b/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__-api/src/main/java/MyService.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ${package};
+
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.controller.ControllerService;
+import org.apache.nifi.processor.exception.ProcessException;
+
+@Tags({"example"})
+@CapabilityDescription("Example Service API.")
+public interface MyService extends ControllerService {
+
+ public void execute() throws ProcessException;
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__-nar/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__-nar/pom.xml b/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__-nar/pom.xml
new file mode 100644
index 0000000..0d8ea56
--- /dev/null
+++ b/nifi-maven-archetypes/nifi-service-bundle-archetype/src/main/resources/archetype-resources/nifi-__artifactBaseName__-nar/pom.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>${groupId}</groupId>
+ <artifactId>${rootArtifactId}</artifactId>
+ <version>${version}</version>
+ </parent>
+
+ <artifactId>nifi-${artifactBaseName}-nar</artifactId>
+ <version>${version}</version>
+ <packaging>nar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>${groupId}</groupId>
+ <artifactId>nifi-${artifactBaseName}-api-nar</artifactId>
+ <version>${version}</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>${groupId}</groupId>
+ <artifactId>nifi-${artifactBaseName}</artifactId>
+ <version>${version}</version>
+ </dependency>
+ </dependencies>
+
+</project>
[42/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-assembly/LICENSE
----------------------------------------------------------------------
diff --git a/nifi-assembly/LICENSE b/nifi-assembly/LICENSE
new file mode 100644
index 0000000..a3a5fdb
--- /dev/null
+++ b/nifi-assembly/LICENSE
@@ -0,0 +1,915 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+APACHE NIFI SUBCOMPONENTS:
+
+The Apache NiFi project contains subcomponents with separate copyright
+notices and license terms. Your use of the source code for the these
+subcomponents is subject to the terms and conditions of the following
+licenses.
+
+This product bundles source from 'Asciidoctor'. Specifically the 'asciidoc-mod.css'.
+The source is available under an MIT LICENSE.
+
+ Copyright (C) 2012-2015 Dan Allen, Ryan Waldron and the Asciidoctor Project
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+
+This product bundles 'Javascript D3 Library' which is available under a
+"3-clause BSD" license.
+
+ Copyright (c) 2010-2014, Michael Bostock
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+ * The name Michael Bostock may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ DISCLAIMED. IN NO EVENT SHALL MICHAEL BOSTOCK BE LIABLE FOR ANY DIRECT,
+ INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+This product bundles 'CodeMirror' which is available under an MIT style license.
+
+ Copyright (C) 2014 by Marijn Haverbeke <ma...@gmail.com> and others
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+
+This product bundles 'JQuery' which is available under and MIT style license.
+ (c) 2005, 2014 jQuery Foundation, Inc.
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+
+This product bundles 'JQuery Event Drag' which is available under an MIT style
+license.
+ Copyright (c) 2008-2015 ThreeDubMedia
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+
+This product bundles 'jQuery Form Plugin' which is available under either the MIT
+or GPL license. The license in effect here is the MIT license
+
+ Copyright 2006-2013 (c) M. Alsup
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+This product bundles 'jQuery UI' which is available under an MIT style license.
+For details see http://jqueryui.com
+
+ Copyright 2014 jQuery Foundation and other contributors
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+This product bundles 'SlickGrid v2.2' which is available under an MIT style license.
+
+ Copyright (c) 2010 Michael Leibman, http://github.com/mleibman/slickgrid
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+This product bundles 'qTip2' which is available under an MIT style license.
+For details see http://qtip2.com
+
+ Copyright (c) 2012 Craig Michael Thompson
+
+ Permission is hereby granted, free of charge, to any person
+ obtaining a copy of this software and associated documentation
+ files (the "Software"), to deal in the Software without
+ restriction, including without limitation the rights to use,
+ copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the
+ Software is furnished to do so, subject to the following
+ conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ OTHER DEALINGS IN THE SOFTWARE.
+
+This product bundles 'jQuery MiniColors' which is available under the MIT License.
+For details see http://www.abeautifulsite.net/
+
+ Copyright Cory LaViska for A Beautiful Site, LLC. (http://www.abeautifulsite.net/)
+
+ Permission is hereby granted, free of charge, to any person
+ obtaining a copy of this software and associated documentation
+ files (the "Software"), to deal in the Software without
+ restriction, including without limitation the rights to use,
+ copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the
+ Software is furnished to do so, subject to the following
+ conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ OTHER DEALINGS IN THE SOFTWARE.
+
+This product bundles 'json2.js' which is available in the 'public domain'.
+ For details see https://github.com/douglascrockford/JSON-js
+
+This product bundles 'reset.css' which is available in the 'public domain'.
+ For details see http://meyerweb.com/eric/tools/css/reset/
+
+This product bundles HexViewJS available under an MIT License
+
+ Copyright (c) 2010 Nick McVeity <nm...@gmail.com>
+
+ Permission is hereby granted, free of charge, to any person
+ obtaining a copy of this software and associated documentation
+ files (the "Software"), to deal in the Software without restriction,
+ including without limitation the rights to use, copy, modify, merge,
+ publish, distribute, sublicense, and/or sell copies of the Software,
+ and to permit persons to whom the Software is furnished to do so,
+ subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+ The binary distribution of this product bundles 'Slf4j' which is available
+ under a "3-clause BSD" license. For details see http://www.slf4j.org/
+
+ Copyright (c) 2004-2013 QOS.ch
+ All rights reserved.
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+ The binary distribution of this product bundles 'Antlr 3' which is available
+ under a "3-clause BSD" license. For details see http://www.antlr3.org/license.html
+
+ Copyright (c) 2010 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+
+ Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+ Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+ Neither the name of the author nor the names of its contributors may be used
+ to endorse or promote products derived from this software without specific
+ prior written permission.
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ THE POSSIBILITY OF SUCH DAMAGE.
+
+ The binary distribution of this product bundles 'Bouncy Castle JDK 1.6 Provider'
+ under an MIT style license.
+
+ Copyright (c) 2000 - 2013 The Legion of the Bouncy Castle Inc. (http://www.bouncycastle.org)
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+
+ The binary distribution of this product bundles 'XMLENC' which is available
+ under a BSD license. More details found here: http://xmlenc.sourceforge.net.
+
+ Copyright 2003-2005, Ernst de Haan <wf...@gmail.com>
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+
+ 1. Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+
+ 2. Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+ 3. Neither the name of the copyright holder nor the names of its contributors
+ may be used to endorse or promote products derived from this software
+ without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS "AS IS"
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ The binary distribution of this product bundles 'Slf4j' which is available under
+ an MIT license.
+
+ Copyright (c) 2004-2013 QOS.ch
+ All rights reserved.
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+ The binary distribution of this product bundles 'ParaNamer' and 'Paranamer Core'
+ which is available under a BSD style license.
+
+ Copyright (c) 2006 Paul Hammant & ThoughtWorks Inc
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. Neither the name of the copyright holders nor the names of its
+ contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ THE POSSIBILITY OF SUCH DAMAGE.
+
+ The binary distribution of this product bundles 'Protocol Buffers - Google's data interchange format'
+ which is available under a BSD style license.
+
+ Copyright 2008 Google Inc. All rights reserved.
+ http://code.google.com/p/protobuf/
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are
+ met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following disclaimer
+ in the documentation and/or other materials provided with the
+ distribution.
+ * Neither the name of Google Inc. nor the names of its
+ contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ The binary distribution of this product bundles 'JCraft Jsch' which is available
+ under a BSD style license.
+ Copyright (c) 2002-2015 Atsuhiko Yamanaka, JCraft,Inc.
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+
+ 1. Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in
+ the documentation and/or other materials provided with the distribution.
+
+ 3. The names of the authors may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES,
+ INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JCRAFT,
+ INC. OR ANY CONTRIBUTORS TO THIS SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
+ OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ The binary distribution of this product bundles 'Scala Library' under a BSD
+ style license.
+
+ Copyright (c) 2002-2015 EPFL
+ Copyright (c) 2011-2015 Typesafe, Inc.
+
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without modification,
+ are permitted provided that the following conditions are met:
+
+ Redistributions of source code must retain the above copyright notice, this list of
+ conditions and the following disclaimer.
+
+ Redistributions in binary form must reproduce the above copyright notice, this list of
+ conditions and the following disclaimer in the documentation and/or other materials
+ provided with the distribution.
+
+ Neither the name of the EPFL nor the names of its contributors may be used to endorse
+ or promote products derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” AND ANY EXPRESS
+ OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+ AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ The binary distribution of this product bundles 'JLine' under a BSD
+ style license.
+
+ Copyright (c) 2002-2006, Marc Prud'hommeaux <mw...@cornell.edu>
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or
+ without modification, are permitted provided that the following
+ conditions are met:
+
+ Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer
+ in the documentation and/or other materials provided with
+ the distribution.
+
+ Neither the name of JLine nor the names of its contributors
+ may be used to endorse or promote products derived from this
+ software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,
+ BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+ AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+ OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+ IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ The binary distribution of this product bundles 'JOpt Simple' under an MIT
+ style license.
+
+ Copyright (c) 2009 Paul R. Holser, Jr.
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+This product bundles 'JCraft Jzlib' which is available under a 3-Clause BSD License.
+
+ Copyright (c) 2002-2014 Atsuhiko Yamanaka, JCraft,Inc.
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+
+ 1. Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in
+ the documentation and/or other materials provided with the distribution.
+
+ 3. The names of the authors may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES,
+ INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JCRAFT,
+ INC. OR ANY CONTRIBUTORS TO THIS SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
+ OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+This product bundles 'asm' which is available under an MIT style license.
+For details see http://asm.ow2.org/asmdex-license.html
+
+ Copyright (c) 2012 France Télécom
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. Neither the name of the copyright holders nor the names of its
+ contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ THE POSSIBILITY OF SUCH DAMAGE.
+
+The binary distribution of this product bundles 'Hamcrest' which is available
+under a BSD license. More details found here: http://hamcrest.org.
+
+ Copyright (c) 2000-2006, www.hamcrest.org
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+
+ Redistributions of source code must retain the above copyright notice, this list of
+ conditions and the following disclaimer. Redistributions in binary form must reproduce
+ the above copyright notice, this list of conditions and the following disclaimer in
+ the documentation and/or other materials provided with the distribution.
+
+ Neither the name of Hamcrest nor the names of its contributors may be used to endorse
+ or promote products derived from this software without specific prior written
+ permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+ EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
+ SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY
+ WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ DAMAGE.
+
+The binary distribution of this product bundles 'leveldbjni-all-1.8.jar' which is available
+ under a BSD style license
+
+ Copyright (c) 2011 FuseSource Corp. All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are
+ met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following disclaimer
+ in the documentation and/or other materials provided with the
+ distribution.
+ * Neither the name of FuseSource Corp. nor the names of its
+ contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+The binary distribution of this product bundles 'Woodstox StAX 2 API' which is
+ "licensed under standard BSD license"
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-assembly/NOTICE
----------------------------------------------------------------------
diff --git a/nifi-assembly/NOTICE b/nifi-assembly/NOTICE
new file mode 100644
index 0000000..118ab49
--- /dev/null
+++ b/nifi-assembly/NOTICE
@@ -0,0 +1,788 @@
+nifi-assembly
+Copyright 2014-2015 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+===========================================
+Apache Software License v2
+===========================================
+
+The following binary components are provided under the Apache Software License v2
+
+ (ASLv2) Apache Commons IO
+ The following NOTICE information applies:
+ Apache Commons IO
+ Copyright 2002-2012 The Apache Software Foundation
+
+ (ASLv2) Apache Commons Net
+ The following NOTICE information applies:
+ Apache Commons Net
+ Copyright 2001-2013 The Apache Software Foundation
+
+ (ASLv2) Apache Commons Collections
+ The following NOTICE information applies:
+ Apache Commons Collections
+ Copyright 2001-2013 The Apache Software Foundation
+
+ (ASLv2) Apache Commons Compress
+ The following NOTICE information applies:
+ Apache Commons Compress
+ Copyright 2002-2014 The Apache Software Foundation
+
+ The files in the package org.apache.commons.compress.archivers.sevenz
+ were derived from the LZMA SDK, version 9.20 (C/ and CPP/7zip/),
+ which has been placed in the public domain:
+
+ "LZMA SDK is placed in the public domain." (http://www.7-zip.org/sdk.html)
+
+ (ASLv2) Jettison
+ The following NOTICE information applies:
+ Copyright 2006 Envoi Solutions LLC
+
+ (ASLv2) Jasypt
+ The following NOTICE information applies:
+ Copyright (c) 2007-2010, The JASYPT team (http://www.jasypt.org)
+
+ (ASLv2) Apache Commons Codec
+ The following NOTICE information applies:
+ Apache Commons Codec
+ Copyright 2002-2014 The Apache Software Foundation
+
+ src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java
+ contains test data from http://aspell.net/test/orig/batch0.tab.
+ Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org)
+
+ ===============================================================================
+
+ The content of package org.apache.commons.codec.language.bm has been translated
+ from the original php source code available at http://stevemorse.org/phoneticinfo.htm
+ with permission from the original authors.
+ Original source copyright:
+ Copyright (c) 2008 Alexander Beider & Stephen P. Morse.
+
+ (ASLv2) Apache HttpComponents
+ The following NOTICE information applies:
+ Apache HttpClient
+ Copyright 1999-2015 The Apache Software Foundation
+
+ Apache HttpCore
+ Copyright 2005-2015 The Apache Software Foundation
+
+ Apache HttpMime
+ Copyright 1999-2013 The Apache Software Foundation
+
+ This project contains annotations derived from JCIP-ANNOTATIONS
+ Copyright (c) 2005 Brian Goetz and Tim Peierls. See http://www.jcip.net
+
+ (ASLv2) Apache Jakarta HttpClient
+ The following NOTICE information applies:
+ Apache Jakarta HttpClient
+ Copyright 1999-2007 The Apache Software Foundation
+
+ (ASLv2) Apache Commons Logging
+ The following NOTICE information applies:
+ Apache Commons Logging
+ Copyright 2003-2014 The Apache Software Foundation
+
+ (ASLv2) Apache Commons Lang
+ The following NOTICE information applies:
+ Apache Commons Lang
+ Copyright 2001-2015 The Apache Software Foundation
+
+ This product includes software from the Spring Framework,
+ under the Apache License 2.0 (see: StringUtils.containsWhitespace())
+
+ (ASLv2) Apache Commons Configuration
+ The following NOTICE information applies:
+ Apache Commons Configuration
+ Copyright 2001-2008 The Apache Software Foundation
+
+ (ASLv2) Apache Commons JEXL
+ The following NOTICE information applies:
+ Apache Commons JEXL
+ Copyright 2001-2011 The Apache Software Foundation
+
+ (ASLv2) Spring Framework
+ The following NOTICE information applies:
+ Spring Framework 4.1.4.RELEASE
+ Copyright (c) 2002-2015 Pivotal, Inc.
+
+ (ASLv2) Apache Flume
+ The following NOTICE information applies:
+ Apache Flume
+ Copyright 2011-2015 Apache Software Foundation
+
+ asynchbase is BSD-licensed software (https://github.com/OpenTSDB/asynchbase)
+
+ async is BSD-licensed software (https://github.com/stumbleupon/async)
+
+ jopt-simple is MIT licensed software (http://pholser.github.io/jopt-simple/license.html)
+
+ scala-library is BSD-like licensed software (http://www.scala-lang.org/license.html)
+
+ (ASLv2) Xalan
+ This product includes software developed by
+ The Apache Software Foundation (http://www.apache.org/).
+
+ Portions of this software was originally based on the following:
+
+ - software copyright (c) 1999-2002, Lotus Development Corporation., http://www.lotus.com.
+ - software copyright (c) 2001-2002, Sun Microsystems., http://www.sun.com.
+ - software copyright (c) 2003, IBM Corporation., http://www.ibm.com.
+ - voluntary contributions made by Ovidiu Predescu (ovidiu@cup.hp.com) on behalf of the
+ Apache Software Foundation and was originally developed at Hewlett Packard Company.
+
+ (ASLv2) Apache XML Commons XML APIs
+ Copyright 2006 The Apache Software Foundation.
+
+ This product includes software developed at
+ The Apache Software Foundation (http://www.apache.org/).
+
+ Portions of this software were originally based on the following:
+ - software copyright (c) 1999, IBM Corporation., http://www.ibm.com.
+ - software copyright (c) 1999, Sun Microsystems., http://www.sun.com.
+ - software copyright (c) 2000 World Wide Web Consortium, http://www.w3.org
+
+ (ASLv2) IRClib
+ The following NOTICE information applies:
+ IRClib -- A Java Internet Relay Chat library --
+ Copyright (C) 2002 - 2006 Christoph Schwering <sc...@gmail.com>
+
+ (ASLv2) Jackson JSON processor
+ The following NOTICE information applies:
+ # Jackson JSON processor
+
+ Jackson is a high-performance, Free/Open Source JSON processing library.
+ It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has
+ been in development since 2007.
+ It is currently developed by a community of developers, as well as supported
+ commercially by FasterXML.com.
+
+ ## Licensing
+
+ Jackson core and extension components may licensed under different licenses.
+ To find the details that apply to this artifact see the accompanying LICENSE file.
+ For more information, including possible other licensing options, contact
+ FasterXML.com (http://fasterxml.com).
+
+ ## Credits
+
+ A list of contributors may be found from CREDITS file, which is included
+ in some artifacts (usually source distributions); but is always available
+ from the source code management (SCM) system project uses.
+
+ (ASLv2) Apache Thrift
+ The following NOTICE information applies:
+ Apache Thrift
+ Copyright 2006-2010 The Apache Software Foundation.
+
+ (ASLv2) Apache MINA
+ The following NOTICE information applies:
+ Apache MINA Core
+ Copyright 2004-2011 Apache MINA Project
+
+ (ASLv2) opencsv (net.sf.opencsv:opencsv:2.3)
+
+ (ASLv2) Apache Velocity
+ The following NOTICE information applies:
+ Apache Velocity
+ Copyright (C) 2000-2007 The Apache Software Foundation
+
+ (ASLv2) ZkClient
+ The following NOTICE information applies:
+ ZkClient
+ Copyright 2009 Stefan Groschupf
+
+ (ASLv2) Apache Commons CLI
+ The following NOTICE information applies:
+ Apache Commons CLI
+ Copyright 2001-2009 The Apache Software Foundation
+
+ (ASLv2) Apache Commons Math
+ The following NOTICE information applies:
+ Apache Commons Math
+ Copyright 2001-2012 The Apache Software Foundation
+
+ This product includes software developed by
+ The Apache Software Foundation (http://www.apache.org/).
+
+ ===============================================================================
+
+ The BracketFinder (package org.apache.commons.math3.optimization.univariate)
+ and PowellOptimizer (package org.apache.commons.math3.optimization.general)
+ classes are based on the Python code in module "optimize.py" (version 0.5)
+ developed by Travis E. Oliphant for the SciPy library (http://www.scipy.org/)
+ Copyright © 2003-2009 SciPy Developers.
+ ===============================================================================
+
+ The LinearConstraint, LinearObjectiveFunction, LinearOptimizer,
+ RelationShip, SimplexSolver and SimplexTableau classes in package
+ org.apache.commons.math3.optimization.linear include software developed by
+ Benjamin McCann (http://www.benmccann.com) and distributed with
+ the following copyright: Copyright 2009 Google Inc.
+ ===============================================================================
+
+ This product includes software developed by the
+ University of Chicago, as Operator of Argonne National
+ Laboratory.
+ The LevenbergMarquardtOptimizer class in package
+ org.apache.commons.math3.optimization.general includes software
+ translated from the lmder, lmpar and qrsolv Fortran routines
+ from the Minpack package
+ Minpack Copyright Notice (1999) University of Chicago. All rights reserved
+ ===============================================================================
+
+ The GraggBulirschStoerIntegrator class in package
+ org.apache.commons.math3.ode.nonstiff includes software translated
+ from the odex Fortran routine developed by E. Hairer and G. Wanner.
+ Original source copyright:
+ Copyright (c) 2004, Ernst Hairer
+ ===============================================================================
+
+ The EigenDecompositionImpl class in package
+ org.apache.commons.math3.linear includes software translated
+ from some LAPACK Fortran routines. Original source copyright:
+ Copyright (c) 1992-2008 The University of Tennessee. All rights reserved.
+ ===============================================================================
+
+ The MersenneTwister class in package org.apache.commons.math3.random
+ includes software translated from the 2002-01-26 version of
+ the Mersenne-Twister generator written in C by Makoto Matsumoto and Takuji
+ Nishimura. Original source copyright:
+ Copyright (C) 1997 - 2002, Makoto Matsumoto and Takuji Nishimura,
+ All rights reserved
+ ===============================================================================
+
+ The LocalizedFormatsTest class in the unit tests is an adapted version of
+ the OrekitMessagesTest class from the orekit library distributed under the
+ terms of the Apache 2 licence. Original source copyright:
+ Copyright 2010 CS Systèmes d'Information
+ ===============================================================================
+
+ The HermiteInterpolator class and its corresponding test have been imported from
+ the orekit library distributed under the terms of the Apache 2 licence. Original
+ source copyright:
+ Copyright 2010-2012 CS Systèmes d'Information
+ ===============================================================================
+
+ The creation of the package "o.a.c.m.analysis.integration.gauss" was inspired
+ by an original code donated by Sébastien Brisard.
+ ===============================================================================
+
+ (ASLv2) Apache log4j
+ The following NOTICE information applies:
+ Apache log4j
+ Copyright 2007 The Apache Software Foundation
+
+ (ASLv2) Apache Tika
+ The following NOTICE information applies:
+ Apache Tika Core
+ Copyright 2007-2015 The Apache Software Foundation
+
+ (ASLv2) Apache Jakarta Commons Digester
+ The following NOTICE information applies:
+ Apache Jakarta Commons Digester
+ Copyright 2001-2006 The Apache Software Foundation
+
+ (ASLv2) Apache Commons BeanUtils
+ The following NOTICE information applies:
+ Apache Commons BeanUtils
+ Copyright 2000-2008 The Apache Software Foundation
+
+ (ASLv2) Apache Avro
+ The following NOTICE information applies:
+ Apache Avro
+ Copyright 2009-2013 The Apache Software Foundation
+
+ (ASLv2) Snappy Java
+ The following NOTICE information applies:
+ This product includes software developed by Google
+ Snappy: http://code.google.com/p/snappy/ (New BSD License)
+
+ This product includes software developed by Apache
+ PureJavaCrc32C from apache-hadoop-common http://hadoop.apache.org/
+ (Apache 2.0 license)
+
+ This library containd statically linked libstdc++. This inclusion is allowed by
+ "GCC RUntime Library Exception"
+ http://gcc.gnu.org/onlinedocs/libstdc++/manual/license.html
+
+ (ASLv2) ApacheDS
+ The following NOTICE information applies:
+ ApacheDS
+ Copyright 2003-2013 The Apache Software Foundation
+
+ (ASLv2) Apache ZooKeeper
+ The following NOTICE information applies:
+ Apache ZooKeeper
+ Copyright 2009-2012 The Apache Software Foundation
+
+ (ASLv2) Apache Commons Daemon
+ The following NOTICE information applies:
+ Apache Commons Daemon
+ Copyright 1999-2013 The Apache Software Foundation
+
+ (ASLv2) Jetty
+ The following NOTICE information applies:
+ Jetty Web Container
+ Copyright 1995-2015 Mort Bay Consulting Pty Ltd.
+
+ (ASLv2) Apache Kafka
+ The following NOTICE information applies:
+ Apache Kafka
+ Copyright 2012 The Apache Software Foundation.
+
+ (ASLv2) Yammer Metrics
+ The following NOTICE information applies:
+ Metrics
+ Copyright 2010-2012 Coda Hale and Yammer, Inc.
+
+ This product includes software developed by Coda Hale and Yammer, Inc.
+
+ This product includes code derived from the JSR-166 project (ThreadLocalRandom), which was released
+ with the following comments:
+
+ Written by Doug Lea with assistance from members of JCP JSR-166
+ Expert Group and released to the public domain, as explained at
+ http://creativecommons.org/publicdomain/zero/1.0/
+
+ (ASLv2) Apache Lucene
+ The following NOTICE information applies:
+ Apache Lucene
+ Copyright 2014 The Apache Software Foundation
+
+ Includes software from other Apache Software Foundation projects,
+ including, but not limited to:
+ - Apache Ant
+ - Apache Jakarta Regexp
+ - Apache Commons
+ - Apache Xerces
+
+ ICU4J, (under analysis/icu) is licensed under an MIT styles license
+ and Copyright (c) 1995-2008 International Business Machines Corporation and others
+
+ Some data files (under analysis/icu/src/data) are derived from Unicode data such
+ as the Unicode Character Database. See http://unicode.org/copyright.html for more
+ details.
+
+ Brics Automaton (under core/src/java/org/apache/lucene/util/automaton) is
+ BSD-licensed, created by Anders Møller. See http://www.brics.dk/automaton/
+
+ The levenshtein automata tables (under core/src/java/org/apache/lucene/util/automaton) were
+ automatically generated with the moman/finenight FSA library, created by
+ Jean-Philippe Barrette-LaPierre. This library is available under an MIT license,
+ see http://sites.google.com/site/rrettesite/moman and
+ http://bitbucket.org/jpbarrette/moman/overview/
+
+ The class org.apache.lucene.util.WeakIdentityMap was derived from
+ the Apache CXF project and is Apache License 2.0.
+
+ The Google Code Prettify is Apache License 2.0.
+ See http://code.google.com/p/google-code-prettify/
+
+ JUnit (junit-4.10) is licensed under the Common Public License v. 1.0
+ See http://junit.sourceforge.net/cpl-v10.html
+
+ This product includes code (JaspellTernarySearchTrie) from Java Spelling Checkin
+ g Package (jaspell): http://jaspell.sourceforge.net/
+ License: The BSD License (http://www.opensource.org/licenses/bsd-license.php)
+
+ The snowball stemmers in
+ analysis/common/src/java/net/sf/snowball
+ were developed by Martin Porter and Richard Boulton.
+ The snowball stopword lists in
+ analysis/common/src/resources/org/apache/lucene/analysis/snowball
+ were developed by Martin Porter and Richard Boulton.
+ The full snowball package is available from
+ http://snowball.tartarus.org/
+
+ The KStem stemmer in
+ analysis/common/src/org/apache/lucene/analysis/en
+ was developed by Bob Krovetz and Sergio Guzman-Lara (CIIR-UMass Amherst)
+ under the BSD-license.
+
+ The Arabic,Persian,Romanian,Bulgarian, and Hindi analyzers (common) come with a default
+ stopword list that is BSD-licensed created by Jacques Savoy. These files reside in:
+ analysis/common/src/resources/org/apache/lucene/analysis/ar/stopwords.txt,
+ analysis/common/src/resources/org/apache/lucene/analysis/fa/stopwords.txt,
+ analysis/common/src/resources/org/apache/lucene/analysis/ro/stopwords.txt,
+ analysis/common/src/resources/org/apache/lucene/analysis/bg/stopwords.txt,
+ analysis/common/src/resources/org/apache/lucene/analysis/hi/stopwords.txt
+ See http://members.unine.ch/jacques.savoy/clef/index.html.
+
+ The German,Spanish,Finnish,French,Hungarian,Italian,Portuguese,Russian and Swedish light stemmers
+ (common) are based on BSD-licensed reference implementations created by Jacques Savoy and
+ Ljiljana Dolamic. These files reside in:
+ analysis/common/src/java/org/apache/lucene/analysis/de/GermanLightStemmer.java
+ analysis/common/src/java/org/apache/lucene/analysis/de/GermanMinimalStemmer.java
+ analysis/common/src/java/org/apache/lucene/analysis/es/SpanishLightStemmer.java
+ analysis/common/src/java/org/apache/lucene/analysis/fi/FinnishLightStemmer.java
+ analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchLightStemmer.java
+ analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchMinimalStemmer.java
+ analysis/common/src/java/org/apache/lucene/analysis/hu/HungarianLightStemmer.java
+ analysis/common/src/java/org/apache/lucene/analysis/it/ItalianLightStemmer.java
+ analysis/common/src/java/org/apache/lucene/analysis/pt/PortugueseLightStemmer.java
+ analysis/common/src/java/org/apache/lucene/analysis/ru/RussianLightStemmer.java
+ analysis/common/src/java/org/apache/lucene/analysis/sv/SwedishLightStemmer.java
+
+ The Stempel analyzer (stempel) includes BSD-licensed software developed
+ by the Egothor project http://egothor.sf.net/, created by Leo Galambos, Martin Kvapil,
+ and Edmond Nolan.
+
+ The Polish analyzer (stempel) comes with a default
+ stopword list that is BSD-licensed created by the Carrot2 project. The file resides
+ in stempel/src/resources/org/apache/lucene/analysis/pl/stopwords.txt.
+ See http://project.carrot2.org/license.html.
+
+ The SmartChineseAnalyzer source code (smartcn) was
+ provided by Xiaoping Gao and copyright 2009 by www.imdict.net.
+
+ WordBreakTestUnicode_*.java (under modules/analysis/common/src/test/)
+ is derived from Unicode data such as the Unicode Character Database.
+ See http://unicode.org/copyright.html for more details.
+
+ The Morfologik analyzer (morfologik) includes BSD-licensed software
+ developed by Dawid Weiss and Marcin Miłkowski (http://morfologik.blogspot.com/).
+
+ Morfologik uses data from Polish ispell/myspell dictionary
+ (http://www.sjp.pl/slownik/en/) licenced on the terms of (inter alia)
+ LGPL and Creative Commons ShareAlike.
+
+ Morfologic includes data from BSD-licensed dictionary of Polish (SGJP)
+ (http://sgjp.pl/morfeusz/)
+
+ Servlet-api.jar and javax.servlet-*.jar are under the CDDL license, the original
+ source code for this can be found at http://www.eclipse.org/jetty/downloads.php
+
+ ===========================================================================
+ Kuromoji Japanese Morphological Analyzer - Apache Lucene Integration
+ ===========================================================================
+
+ This software includes a binary and/or source version of data from
+
+ mecab-ipadic-2.7.0-20070801
+
+ which can be obtained from
+
+ http://atilika.com/releases/mecab-ipadic/mecab-ipadic-2.7.0-20070801.tar.gz
+
+ or
+
+ http://jaist.dl.sourceforge.net/project/mecab/mecab-ipadic/2.7.0-20070801/mecab-ipadic-2.7.0-20070801.tar.gz
+
+ ===========================================================================
+ mecab-ipadic-2.7.0-20070801 Notice
+ ===========================================================================
+
+ Nara Institute of Science and Technology (NAIST),
+ the copyright holders, disclaims all warranties with regard to this
+ software, including all implied warranties of merchantability and
+ fitness, in no event shall NAIST be liable for
+ any special, indirect or consequential damages or any damages
+ whatsoever resulting from loss of use, data or profits, whether in an
+ action of contract, negligence or other tortuous action, arising out
+ of or in connection with the use or performance of this software.
+
+ A large portion of the dictionary entries
+ originate from ICOT Free Software. The following conditions for ICOT
+ Free Software applies to the current dictionary as well.
+
+ Each User may also freely distribute the Program, whether in its
+ original form or modified, to any third party or parties, PROVIDED
+ that the provisions of Section 3 ("NO WARRANTY") will ALWAYS appear
+ on, or be attached to, the Program, which is distributed substantially
+ in the same form as set out herein and that such intended
+ distribution, if actually made, will neither violate or otherwise
+ contravene any of the laws and regulations of the countries having
+ jurisdiction over the User or the intended distribution itself.
+
+ NO WARRANTY
+
+ The program was produced on an experimental basis in the course of the
+ research and development conducted during the project and is provided
+ to users as so produced on an experimental basis. Accordingly, the
+ program is provided without any warranty whatsoever, whether express,
+ implied, statutory or otherwise. The term "warranty" used herein
+ includes, but is not limited to, any warranty of the quality,
+ performance, merchantability and fitness for a particular purpose of
+ the program and the nonexistence of any infringement or violation of
+ any right of any third party.
+
+ Each user of the program will agree and understand, and be deemed to
+ have agreed and understood, that there is no warranty whatsoever for
+ the program and, accordingly, the entire risk arising from or
+ otherwise connected with the program is assumed by the user.
+
+ Therefore, neither ICOT, the copyright holder, or any other
+ organization that participated in or was otherwise related to the
+ development of the program and their respective officials, directors,
+ officers and other employees shall be held liable for any and all
+ damages, including, without limitation, general, special, incidental
+ and consequential damages, arising out of or otherwise in connection
+ with the use or inability to use the program or any product, material
+ or result produced or otherwise obtained by using the program,
+ regardless of whether they have been advised of, or otherwise had
+ knowledge of, the possibility of such damages at any time during the
+ project or thereafter. Each user will be deemed to have agreed to the
+ foregoing by his or her commencement of use of the program. The term
+ "use" as used herein includes, but is not limited to, the use,
+ modification, copying and distribution of the program and the
+ production of secondary products from the program.
+
+ In the case where the program, whether in its original form or
+ modified, was distributed or delivered to or received by a user from
+ any person, organization or entity other than ICOT, unless it makes or
+ grants independently of ICOT any specific warranty to the user in
+ writing, such person, organization or entity, will also be exempted
+ from and not be held liable to the user for any such damages as noted
+ above as far as the program is concerned.
+
+ (ASLv2) Joda Time
+ The following NOTICE information applies:
+ This product includes software developed by
+ Joda.org (http://www.joda.org/).
+
+ (ASLv2) Apache ActiveMQ
+ The following NOTICE information applies:
+ ActiveMQ :: Client
+ Copyright 2005-2015 The Apache Software Foundation
+
+ (ASLv2) Apache Geronimo
+ The following NOTICE information applies:
+ Apache Geronimo
+ Copyright 2003-2008 The Apache Software Foundation
+
+ (ASLv2) Swagger Core
+ The following NOTICE information applies:
+ Swagger Core 1.5.3-M1
+ Copyright 2015 Reverb Technologies, Inc.
+
+ (ASLv2) JSON-SMART
+ The following NOTICE information applies:
+ Copyright 2011 JSON-SMART authors
+
+ (ASLv2) JsonPath
+ The following NOTICE information applies:
+ Copyright 2011 JsonPath authors
+
+ (ASLv2) Kite SDK
+ The following NOTICE information applies:
+ This product includes software developed by Cloudera, Inc.
+ (http://www.cloudera.com/).
+
+ This product includes software developed at
+ The Apache Software Foundation (http://www.apache.org/).
+
+ This product includes software developed by
+ Saxonica (http://www.saxonica.com/).
+
+ (ASLv2) MongoDB Java Driver
+ The following NOTICE information applies:
+ Copyright (C) 2008-2013 10gen, Inc.
+
+ (ASLv2) Parquet MR
+ The following NOTICE information applies:
+ Parquet MR
+ Copyright 2012 Twitter, Inc.
+
+ This project includes code from https://github.com/lemire/JavaFastPFOR
+ parquet-column/src/main/java/parquet/column/values/bitpacking/LemireBitPacking.java
+ Apache License Version 2.0 http://www.apache.org/licenses/.
+ (c) Daniel Lemire, http://lemire.me/en/
+
+ (ASLv2) Twitter4J
+ The following NOTICE information applies:
+ Copyright 2007 Yusuke Yamamoto
+
+ Twitter4J includes software from JSON.org to parse JSON response from the Twitter API. You can see the license term at http://www.JSON.org/license.html
+
+ (ASLv2) JOAuth
+ The following NOTICE information applies:
+ JOAuth
+ Copyright 2010-2013 Twitter, Inc
+
+ (ASLv2) Hosebird Client
+ The following NOTICE information applies:
+ Hosebird Client (hbc)
+ Copyright 2013 Twitter, Inc.
+
+ (ASLv2) GeoIP2 Java API
+ The following NOTICE information applies:
+ GeoIP2 Java API
+ This software is Copyright (c) 2013 by MaxMind, Inc.
+
+ (ASLv2) Woodstox Core ASL
+ The following NOTICE information applies:
+ This product currently only contains code developed by authors
+ of specific components, as identified by the source code files.
+
+ Since product implements StAX API, it has dependencies to StAX API
+ classes.
+
+ (ASLv2) Amazon Web Services SDK
+ The following NOTICE information applies:
+ Copyright 2010-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
+ This product includes software developed by
+ Amazon Technologies, Inc (http://www.amazon.com/).
+
+ **********************
+ THIRD PARTY COMPONENTS
+ **********************
+ This software includes third party software subject to the following copyrights:
+ - XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty.
+ - JSON parsing and utility functions from JSON.org - Copyright 2002 JSON.org.
+ - PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc.
+
+ (ASLv2) Apache Commons DBCP
+ The following NOTICE information applies:
+ Apache Commons DBCP
+ Copyright 2001-2015 The Apache Software Foundation.
+
+ (ASLv2) Apache Commons Pool
+ The following NOTICE information applies:
+ Apache Commons Pool
+ Copyright 1999-2009 The Apache Software Foundation.
+
+ (ASLv2) Apache Derby
+ The following NOTICE information applies:
+ Apache Derby
+ Copyright 2004-2014 Apache, Apache DB, Apache Derby, Apache Torque, Apache JDO, Apache DDLUtils,
+ the Derby hat logo, the Apache JDO logo, and the Apache feather logo are trademarks of The Apache Software Foundation.
+
+ (ASLv2) Apache Directory Server
+ The following NOTICE information applies:
+ ApacheDS Protocol Kerberos Codec
+ Copyright 2003-2013 The Apache Software Foundation
+
+ ApacheDS I18n
+ Copyright 2003-2013 The Apache Software Foundation
+
+ Apache Directory API ASN.1 API
+ Copyright 2003-2013 The Apache Software Foundation
+
+ Apache Directory LDAP API Utilities
+ Copyright 2003-2013 The Apache Software Foundation
+
+ (ASLv2) Apache Curator
+ The following NOTICE information applies:
+ Curator Framework
+ Copyright 2011-2014 The Apache Software Foundation
+
+ Curator Client
+ Copyright 2011-2014 The Apache Software Foundation
+
+ Curator Recipes
+ Copyright 2011-2014 The Apache Software Foundation
+
+ (ASLv2) The Netty Project
+ The following NOTICE information applies:
+ The Netty Project
+ Copyright 2011 The Netty Project
+
+ (ASLv2) Apache Xerces Java
+ The following NOTICE information applies:
+ Apache Xerces Java
+ Copyright 1999-2007 The Apache Software Foundation
+
+ This product includes software developed at
+ The Apache Software Foundation (http://www.apache.org/).
+
+ Portions of this software were originally based on the following:
+ - software copyright (c) 1999, IBM Corporation., http://www.ibm.com.
+ - software copyright (c) 1999, Sun Microsystems., http://www.sun.com.
+ - voluntary contributions made by Paul Eng on behalf of the
+ Apache Software Foundation that were originally developed at iClick, Inc.,
+ software copyright (c) 1999.
+
+ (ASLv2) Google Guice
+ The following NOTICE information applies:
+ Google Guice - Core Library
+ Copyright 2006-2011 Google, Inc.
+
+ Google Guice - Extensions - Servlet
+ Copyright 2006-2011 Google, Inc.
+
+************************
+Common Development and Distribution License 1.1
+************************
+
+The following binary components are provided under the Common Development and Distribution License 1.1. See project link for details.
+
+ (CDDL 1.1) (GPL2 w/ CPE) jersey-client (com.sun.jersey:jersey-client:jar:1.19 - https://jersey.java.net/jersey-client/)
+ (CDDL 1.1) (GPL2 w/ CPE) jersey-core (com.sun.jersey:jersey-core:jar:1.19 - https://jersey.java.net/jersey-core/)
+ (CDDL 1.1) (GPL2 w/ CPE) jersey-spring (com.sun.jersey:jersey-spring:jar:1.19 - https://jersey.java.net/jersey-spring/)
+ (CDDL 1.1) (GPL2 w/ CPE) jersey-servlet (com.sun.jersey:jersey-servlet:jar:1.19 - https://jersey.java.net/jersey-servlet/)
+ (CDDL 1.1) (GPL2 w/ CPE) jersey-multipart (com.sun.jersey:jersey-multipart:jar:1.19 - https://jersey.java.net/jersey-multipart/)
+ (CDDL 1.1) (GPL2 w/ CPE) jersey-server (com.sun.jersey:jersey-server:jar:1.19 - https://jersey.java.net/jersey-server/)
+ (CDDL 1.1) (GPL2 w/ CPE) jersey-json (com.sun.jersey:jersey-json:jar:1.19 - https://jersey.java.net/jersey-json/)
+ (CDDL 1.1) (GPL2 w/ CPE) Old JAXB Runtime (com.sun.xml.bind:jaxb-impl:jar:2.2.3-1 - http://jaxb.java.net/)
+ (CDDL 1.1) (GPL2 w/ CPE) Java Architecture For XML Binding (javax.xml.bind:jaxb-api:jar:2.2.2 - https://jaxb.dev.java.net/)
+ (CDDL 1.1) (GPL2 w/ CPE) MIME Streaming Extension (org.jvnet.mimepull:mimepull:jar:1.9.3 - http://mimepull.java.net)
+ (CDDL 1.1) (GPL2 w/ CPE) JavaMail API (compat) (javax.mail:mail:jar:1.4.7 - http://kenai.com/projects/javamail/mail)
+ (CDDL 1.1) (GPL2 w/ CPE) JSP Implementation (org.glassfish.web:javax.servlet.jsp:jar:2.3.2 - http://jsp.java.net)
+ (CDDL 1.1) (GPL2 w/ CPE) JavaServer Pages (TM) TagLib Implementation (org.glassfish.web:javax.servlet.jsp.jstl:jar:1.2.2 - http://jstl.java.net)
+ (CDDL 1.1) (GPL2 w/ CPE) Expression Language 3.0 (org.glassfish:javax.el:jar:3.0.0 - http://el-spec.java.net)
+ (CDDL 1.1) (GPL2 w/ CPE) JavaServer Pages(TM) API (javax.servlet.jsp:javax.servlet.jsp-api:jar:2.3.1 - http://jsp.java.net)
+ (CDDL 1.1) (GPL2 w/ CPE) Expression Language 3.0 API (javax.el:javax.el-api:jar:3.0.0 - http://uel-spec.java.net)
+ (CDDL 1.1) (GPL2 w/ CPE) JavaServer Pages(TM) Standard Tag Library API (javax.servlet.jsp.jstl:javax.servlet.jsp.jstl-api:jar:1.2.1 - http://jcp.org/en/jsr/detail?id=52)
+ (CDDL 1.1) (GPL2 w/ CPE) Java Servlet API (javax.servlet:javax.servlet-api:jar:3.1.0 - http://servlet-spec.java.net)
+ (CDDL 1.1) (GPL2 w/ CPE) Javax JMS Api (javax.jms:javax.jms-api:jar:2.0.1 - http://java.net/projects/jms-spec/pages/Home)
+
+************************
+Common Development and Distribution License 1.0
+************************
+
+The following binary components are provided under the Common Development and Distribution License 1.0. See project link for details.
+
+ (CDDL 1.0) JavaServlet(TM) Specification (javax.servlet:servlet-api:jar:2.5 - no url available)
+ (CDDL 1.0) (GPL3) Streaming API For XML (javax.xml.stream:stax-api:jar:1.0-2 - no url provided)
+ (CDDL 1.0) JavaBeans Activation Framework (JAF) (javax.activation:activation:jar:1.1 - http://java.sun.com/products/javabeans/jaf/index.jsp)
+ (CDDL 1.0) JavaServer Pages(TM) API (javax.servlet.jsp:jsp-api:jar:2.1 - http://jsp.java.net)
+ (CDDL 1.0) JSR311 API (javax.ws.rs:jsr311-api:jar:1.1.1 - https://jsr311.dev.java.net)
+
+************************
+Creative Commons Attribution-ShareAlike 3.0
+************************
+
+The following binary components are provided under the Creative Commons Attribution-ShareAlike 3.0. See project link for details.
+
+ (CCAS 3.0) MaxMind DB (https://github.com/maxmind/MaxMind-DB)
+
+************************
+Eclipse Public License 1.0
+************************
+
+The following binary components are provided under the Eclipse Public License 1.0. See project link for details.
+
+ (EPL 1.0) AspectJ Weaver (org.aspectj:aspectjweaver:jar:1.8.5 - http://www.aspectj.org)
+ (EPL 1.0)(MPL 2.0) H2 Database (com.h2database:h2:jar:1.3.176 - http://www.h2database.com/html/license.html)
+ (EPL 1.0)(LGPL 2.1) Logback Classic (ch.qos.logback:logback-classic:jar:1.1.3 - http://logback.qos.ch/)
+ (EPL 1.0)(LGPL 2.1) Logback Core (ch.qos.logback:logback-core:jar:1.1.3 - http://logback.qos.ch/)
+
+*****************
+Mozilla Public License v2.0
+*****************
+
+The following binary components are provided under the Mozilla Public License v2.0. See project link for details.
+
+ (MPL 2.0) Saxon HE (net.sf.saxon:Saxon-HE:jar:9.6.0-5 - http://www.saxonica.com/)
+
+*****************
+Mozilla Public License v1.1
+*****************
+
+The following binary components are provided under the Mozilla Public License v1.1. See project link for details.
+
+ (MPL 1.1) HAPI Base (ca.uhn.hapi:hapi-base:2.2 - http://hl7api.sourceforge.net/)
+ (MPL 1.1) HAPI Structures (ca.uhn.hapi:hapi-structures-v*:2.2 - http://hl7api.sourceforge.net/)
+
+*****************
+Public Domain
+*****************
+
+The following binary components are provided to the 'Public Domain'. See project link for details.
+
+ (Public Domain) XZ for Java (org.tukaani:xz:jar:1.5 - http://tukaani.org/xz/java.html
+ (Public Domain) AOP Alliance 1.0 (http://aopalliance.sourceforge.net/)
\ No newline at end of file
[20/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-write-ahead-log/src/main/java/org/wali/MinimalLockingWriteAheadLog.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-write-ahead-log/src/main/java/org/wali/MinimalLockingWriteAheadLog.java b/nifi-commons/nifi-write-ahead-log/src/main/java/org/wali/MinimalLockingWriteAheadLog.java
new file mode 100644
index 0000000..a5889ed
--- /dev/null
+++ b/nifi-commons/nifi-write-ahead-log/src/main/java/org/wali/MinimalLockingWriteAheadLog.java
@@ -0,0 +1,1028 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.wali;
+
+import static java.util.Objects.requireNonNull;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.channels.FileChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CopyOnWriteArraySet;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.regex.Pattern;
+
+import org.apache.nifi.stream.io.BufferedInputStream;
+import org.apache.nifi.stream.io.BufferedOutputStream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * <p>
+ * This implementation provides as little Locking as possible in order to
+ * provide the highest throughput possible. However, this implementation is ONLY
+ * appropriate if it can be guaranteed that only a single thread will ever issue
+ * updates for a given Record at any one time.
+ * </p>
+ *
+ * @param <T> type of record this WAL is for
+ */
+public final class MinimalLockingWriteAheadLog<T> implements WriteAheadRepository<T> {
+
+ private final Path basePath;
+ private final Path partialPath;
+ private final Path snapshotPath;
+
+ private final SerDe<T> serde;
+ private final SyncListener syncListener;
+ private final FileChannel lockChannel;
+ private final AtomicLong transactionIdGenerator = new AtomicLong(0L);
+
+ private final Partition<T>[] partitions;
+ private final AtomicLong partitionIndex = new AtomicLong(0L);
+ private final ConcurrentMap<Object, T> recordMap = new ConcurrentHashMap<>();
+ private final Map<Object, T> unmodifiableRecordMap = Collections.unmodifiableMap(recordMap);
+ private final Set<String> externalLocations = new CopyOnWriteArraySet<>();
+
+ private final Set<String> recoveredExternalLocations = new CopyOnWriteArraySet<>();
+
+ private final AtomicInteger numberBlackListedPartitions = new AtomicInteger(0);
+
+ private static final Logger logger = LoggerFactory.getLogger(MinimalLockingWriteAheadLog.class);
+
+ private final ReadWriteLock rwLock = new ReentrantReadWriteLock();
+ private final Lock readLock = rwLock.readLock(); // required to update a partition
+ private final Lock writeLock = rwLock.writeLock(); // required for checkpoint
+
+ private volatile boolean updated = false;
+ private volatile boolean recovered = false;
+
+ public MinimalLockingWriteAheadLog(final Path path, final int partitionCount, final SerDe<T> serde, final SyncListener syncListener) throws IOException {
+ this(new TreeSet<>(Collections.singleton(path)), partitionCount, serde, syncListener);
+ }
+
+ /**
+ *
+ * @param paths a sorted set of Paths to use for the partitions/journals and
+ * the snapshot. The snapshot will always be written to the first path
+ * specified.
+ * @param partitionCount the number of partitions/journals to use. For best
+ * performance, this should be close to the number of threads that are
+ * expected to update the repository simultaneously
+ * @param serde the serializer/deserializer for records
+ * @param syncListener the listener
+ * @throws IOException if unable to initialize due to IO issue
+ */
+ @SuppressWarnings("unchecked")
+ public MinimalLockingWriteAheadLog(final SortedSet<Path> paths, final int partitionCount, final SerDe<T> serde, final SyncListener syncListener) throws IOException {
+ this.syncListener = syncListener;
+
+ requireNonNull(paths);
+ requireNonNull(serde);
+
+ if (paths.isEmpty()) {
+ throw new IllegalArgumentException("Paths must be non-empty");
+ }
+
+ int existingPartitions = 0;
+ for (final Path path : paths) {
+ if (!Files.exists(path)) {
+ Files.createDirectories(path);
+ }
+
+ final File file = path.toFile();
+ if (!file.isDirectory()) {
+ throw new IOException("Path given [" + path + "] is not a directory");
+ }
+ if (!file.canWrite()) {
+ throw new IOException("Path given [" + path + "] is not writable");
+ }
+ if (!file.canRead()) {
+ throw new IOException("Path given [" + path + "] is not readable");
+ }
+ if (!file.canExecute()) {
+ throw new IOException("Path given [" + path + "] is not executable");
+ }
+
+ final File[] children = file.listFiles();
+ if (children != null) {
+ for (final File child : children) {
+ if (child.isDirectory() && child.getName().startsWith("partition-")) {
+ existingPartitions++;
+ }
+ }
+
+ if (existingPartitions != 0 && existingPartitions != partitionCount) {
+ logger.warn("Constructing MinimalLockingWriteAheadLog with partitionCount={}, but the repository currently has "
+ + "{} partitions; ignoring argument and proceeding with {} partitions",
+ new Object[]{partitionCount, existingPartitions, existingPartitions});
+ }
+ }
+ }
+
+ this.basePath = paths.iterator().next();
+ this.partialPath = basePath.resolve("snapshot.partial");
+ this.snapshotPath = basePath.resolve("snapshot");
+ this.serde = serde;
+
+ final Path lockPath = basePath.resolve("wali.lock");
+ lockChannel = new FileOutputStream(lockPath.toFile()).getChannel();
+ lockChannel.lock();
+
+ partitions = new Partition[partitionCount];
+
+ Iterator<Path> pathIterator = paths.iterator();
+ for (int i = 0; i < partitionCount; i++) {
+ // If we're out of paths, create a new iterator to start over.
+ if (!pathIterator.hasNext()) {
+ pathIterator = paths.iterator();
+ }
+
+ final Path partitionBasePath = pathIterator.next();
+
+ partitions[i] = new Partition<>(partitionBasePath.resolve("partition-" + i), serde, i, getVersion());
+ }
+ }
+
+ @Override
+ public int update(final Collection<T> records, final boolean forceSync) throws IOException {
+ if (!recovered) {
+ throw new IllegalStateException("Cannot update repository until record recovery has been performed");
+ }
+
+ if (records.isEmpty()) {
+ return -1;
+ }
+
+ updated = true;
+ readLock.lock();
+ try {
+ while (true) {
+ final int numBlackListed = numberBlackListedPartitions.get();
+ if (numBlackListed >= partitions.length) {
+ throw new IOException("All Partitions have been blacklisted due to "
+ + "failures when attempting to update. If the Write-Ahead Log is able to perform a checkpoint, "
+ + "this issue may resolve itself. Otherwise, manual intervention will be required.");
+ }
+
+ final long partitionIdx = partitionIndex.getAndIncrement();
+ final int resolvedIdx = (int) (partitionIdx % partitions.length);
+ final Partition<T> partition = partitions[resolvedIdx];
+ if (partition.tryClaim()) {
+ try {
+ final long transactionId = transactionIdGenerator.getAndIncrement();
+ if (logger.isTraceEnabled()) {
+ for (final T record : records) {
+ logger.trace("Partition {} performing Transaction {}: {}", new Object[]{partition, transactionId, record});
+ }
+ }
+
+ try {
+ partition.update(records, transactionId, unmodifiableRecordMap, forceSync);
+ } catch (final Throwable t) {
+ partition.blackList();
+ numberBlackListedPartitions.incrementAndGet();
+ throw t;
+ }
+
+ if (forceSync && syncListener != null) {
+ syncListener.onSync(resolvedIdx);
+ }
+ } finally {
+ partition.releaseClaim();
+ }
+
+ for (final T record : records) {
+ final UpdateType updateType = serde.getUpdateType(record);
+ final Object recordIdentifier = serde.getRecordIdentifier(record);
+
+ if (updateType == UpdateType.DELETE) {
+ recordMap.remove(recordIdentifier);
+ } else if (updateType == UpdateType.SWAP_OUT) {
+ final String newLocation = serde.getLocation(record);
+ if (newLocation == null) {
+ logger.error("Received Record (ID=" + recordIdentifier + ") with UpdateType of SWAP_OUT but "
+ + "no indicator of where the Record is to be Swapped Out to; these records may be "
+ + "lost when the repository is restored!");
+ } else {
+ recordMap.remove(recordIdentifier);
+ this.externalLocations.add(newLocation);
+ }
+ } else if (updateType == UpdateType.SWAP_IN) {
+ final String newLocation = serde.getLocation(record);
+ if (newLocation == null) {
+ logger.error("Received Record (ID=" + recordIdentifier + ") with UpdateType of SWAP_IN but no "
+ + "indicator of where the Record is to be Swapped In from; these records may be duplicated "
+ + "when the repository is restored!");
+ } else {
+ externalLocations.remove(newLocation);
+ }
+ recordMap.put(recordIdentifier, record);
+ } else {
+ recordMap.put(recordIdentifier, record);
+ }
+ }
+
+ return resolvedIdx;
+ }
+ }
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public Collection<T> recoverRecords() throws IOException {
+ if (updated) {
+ throw new IllegalStateException("Cannot recover records after updating the repository; must call recoverRecords first");
+ }
+
+ final long recoverStart = System.nanoTime();
+ writeLock.lock();
+ try {
+ Long maxTransactionId = recoverFromSnapshot(recordMap);
+ recoverFromEdits(recordMap, maxTransactionId);
+
+ for (final Partition<T> partition : partitions) {
+ final long transId = partition.getMaxRecoveredTransactionId();
+ if (maxTransactionId == null || transId > maxTransactionId) {
+ maxTransactionId = transId;
+ }
+ }
+
+ this.transactionIdGenerator.set(maxTransactionId + 1);
+ this.externalLocations.addAll(recoveredExternalLocations);
+ logger.info("{} finished recovering records. Performing Checkpoint to ensure proper state of Partitions before updates", this);
+ } finally {
+ writeLock.unlock();
+ }
+ final long recoverNanos = System.nanoTime() - recoverStart;
+ final long recoveryMillis = TimeUnit.MILLISECONDS.convert(recoverNanos, TimeUnit.NANOSECONDS);
+ logger.info("Successfully recovered {} records in {} milliseconds", recordMap.size(), recoveryMillis);
+ checkpoint();
+
+ recovered = true;
+ return recordMap.values();
+ }
+
+ @Override
+ public Set<String> getRecoveredSwapLocations() throws IOException {
+ return recoveredExternalLocations;
+ }
+
+ private Long recoverFromSnapshot(final Map<Object, T> recordMap) throws IOException {
+ final boolean partialExists = Files.exists(partialPath);
+ final boolean snapshotExists = Files.exists(snapshotPath);
+
+ if (!partialExists && !snapshotExists) {
+ return null;
+ }
+
+ if (partialExists && snapshotExists) {
+ // both files exist -- assume we failed while checkpointing. Delete
+ // the partial file
+ Files.delete(partialPath);
+ } else if (partialExists) {
+ // partial exists but snapshot does not -- we must have completed
+ // creating the partial, deleted the snapshot
+ // but crashed before renaming the partial to the snapshot. Just
+ // rename partial to snapshot
+ Files.move(partialPath, snapshotPath);
+ }
+
+ if (Files.size(snapshotPath) == 0) {
+ logger.warn("{} Found 0-byte Snapshot file; skipping Snapshot file in recovery", this);
+ return null;
+ }
+
+ // at this point, we know the snapshotPath exists because if it didn't, then we either returned null
+ // or we renamed partialPath to snapshotPath. So just Recover from snapshotPath.
+ try (final DataInputStream dataIn = new DataInputStream(new BufferedInputStream(Files.newInputStream(snapshotPath, StandardOpenOption.READ)))) {
+ final String waliImplementationClass = dataIn.readUTF();
+ final int waliImplementationVersion = dataIn.readInt();
+
+ if (!waliImplementationClass.equals(MinimalLockingWriteAheadLog.class.getName())) {
+ throw new IOException("Write-Ahead Log located at " + snapshotPath + " was written using the "
+ + waliImplementationClass + " class; cannot restore using " + getClass().getName());
+ }
+
+ if (waliImplementationVersion > getVersion()) {
+ throw new IOException("Write-Ahead Log located at " + snapshotPath + " was written using version "
+ + waliImplementationVersion + " of the " + waliImplementationClass + " class; cannot restore using Version " + getVersion());
+ }
+
+ dataIn.readUTF(); // ignore serde class name for now
+ final int serdeVersion = dataIn.readInt();
+ final long maxTransactionId = dataIn.readLong();
+ final int numRecords = dataIn.readInt();
+
+ for (int i = 0; i < numRecords; i++) {
+ final T record = serde.deserializeRecord(dataIn, serdeVersion);
+ if (record == null) {
+ throw new EOFException();
+ }
+
+ final UpdateType updateType = serde.getUpdateType(record);
+ if (updateType == UpdateType.DELETE) {
+ logger.warn("While recovering from snapshot, found record with type 'DELETE'; this record will not be restored");
+ continue;
+ }
+
+ logger.trace("Recovered from snapshot: {}", record);
+ recordMap.put(serde.getRecordIdentifier(record), record);
+ }
+
+ final int numSwapRecords = dataIn.readInt();
+ final Set<String> swapLocations = new HashSet<>();
+ for (int i = 0; i < numSwapRecords; i++) {
+ swapLocations.add(dataIn.readUTF());
+ }
+ this.recoveredExternalLocations.addAll(swapLocations);
+
+ logger.debug("{} restored {} Records and {} Swap Files from Snapshot, ending with Transaction ID {}",
+ new Object[]{this, numRecords, recoveredExternalLocations.size(), maxTransactionId});
+ return maxTransactionId;
+ }
+ }
+
+ /**
+ * Recovers records from the edit logs via the Partitions. Returns a boolean
+ * if recovery of a Partition requires the Write-Ahead Log be checkpointed
+ * before modification.
+ *
+ * @param modifiableRecordMap map
+ * @param maxTransactionIdRestored index of max restored transaction
+ * @throws IOException if unable to recover from edits
+ */
+ private void recoverFromEdits(final Map<Object, T> modifiableRecordMap, final Long maxTransactionIdRestored) throws IOException {
+ final Map<Object, T> updateMap = new HashMap<>();
+ final Map<Object, T> unmodifiableRecordMap = Collections.unmodifiableMap(modifiableRecordMap);
+ final Map<Object, T> ignorableMap = new HashMap<>();
+ final Set<String> ignorableSwapLocations = new HashSet<>();
+
+ // populate a map of the next transaction id for each partition to the
+ // partition that has that next transaction id.
+ final SortedMap<Long, Partition<T>> transactionMap = new TreeMap<>();
+ for (final Partition<T> partition : partitions) {
+ Long transactionId;
+ boolean keepTransaction;
+ do {
+ transactionId = partition.getNextRecoverableTransactionId();
+
+ keepTransaction = transactionId == null || maxTransactionIdRestored == null || transactionId > maxTransactionIdRestored;
+ if (keepTransaction && transactionId != null) {
+ // map this transaction id to its partition so that we can
+ // start restoring transactions from this partition,
+ // starting at 'transactionId'
+ transactionMap.put(transactionId, partition);
+ } else if (transactionId != null) {
+ // skip the next transaction, because our snapshot already
+ // contained this transaction.
+ try {
+ partition.recoverNextTransaction(ignorableMap, updateMap, ignorableSwapLocations);
+ } catch (final EOFException e) {
+ logger.error("{} unexpectedly reached End of File while reading from {} for Transaction {}; "
+ + "assuming crash and ignoring this transaction.",
+ new Object[]{this, partition, transactionId});
+ }
+ }
+ } while (!keepTransaction);
+ }
+
+ while (!transactionMap.isEmpty()) {
+ final Map.Entry<Long, Partition<T>> firstEntry = transactionMap.entrySet().iterator().next();
+ final Long firstTransactionId = firstEntry.getKey();
+ final Partition<T> nextPartition = firstEntry.getValue();
+
+ try {
+ updateMap.clear();
+ final Set<Object> idsRemoved = nextPartition.recoverNextTransaction(unmodifiableRecordMap, updateMap, recoveredExternalLocations);
+ modifiableRecordMap.putAll(updateMap);
+ for (final Object id : idsRemoved) {
+ modifiableRecordMap.remove(id);
+ }
+ } catch (final EOFException e) {
+ logger.error("{} unexpectedly reached End-of-File when reading from {} for Transaction ID {}; "
+ + "assuming crash and ignoring this transaction",
+ new Object[]{this, nextPartition, firstTransactionId});
+ }
+
+ transactionMap.remove(firstTransactionId);
+
+ Long subsequentTransactionId = null;
+ try {
+ subsequentTransactionId = nextPartition.getNextRecoverableTransactionId();
+ } catch (final IOException e) {
+ logger.error("{} unexpectedly found End-of-File when reading from {} for Transaction ID {}; "
+ + "assuming crash and ignoring this transaction",
+ new Object[]{this, nextPartition, firstTransactionId});
+ }
+
+ if (subsequentTransactionId != null) {
+ transactionMap.put(subsequentTransactionId, nextPartition);
+ }
+ }
+
+ for (final Partition<T> partition : partitions) {
+ partition.endRecovery();
+ }
+ }
+
+ @Override
+ public synchronized int checkpoint() throws IOException {
+ final Set<T> records;
+ final Set<String> swapLocations;
+ final long maxTransactionId;
+
+ final long startNanos = System.nanoTime();
+
+ FileOutputStream fileOut = null;
+ DataOutputStream dataOut = null;
+
+ long stopTheWorldNanos = -1L;
+ long stopTheWorldStart = -1L;
+ try {
+ writeLock.lock();
+ try {
+ stopTheWorldStart = System.nanoTime();
+ // stop the world while we make a copy of the records that must
+ // be checkpointed and rollover the partitions.
+ // We copy the records because serializing them is potentially
+ // very expensive, especially when we have hundreds
+ // of thousands or even millions of them. We don't want to
+ // prevent WALI from being used during this time.
+
+ // So the design is to copy all of the records, determine the
+ // last transaction ID that the records represent,
+ // and roll over the partitions to new write-ahead logs.
+ // Then, outside of the write lock, we will serialize the data
+ // to disk, and then remove the old Partition data.
+ records = new HashSet<>(recordMap.values());
+ maxTransactionId = transactionIdGenerator.get() - 1;
+
+ swapLocations = new HashSet<>(externalLocations);
+ for (final Partition<T> partition : partitions) {
+ try {
+ partition.rollover();
+ } catch (final Throwable t) {
+ partition.blackList();
+ numberBlackListedPartitions.getAndIncrement();
+ throw t;
+ }
+ }
+
+ // notify global sync with the write lock held. We do this because we don't want the repository to get updated
+ // while the listener is performing its necessary tasks
+ if (syncListener != null) {
+ syncListener.onGlobalSync();
+ }
+ } finally {
+ writeLock.unlock();
+ }
+
+ stopTheWorldNanos = System.nanoTime() - stopTheWorldStart;
+
+ // perform checkpoint, writing to .partial file
+ fileOut = new FileOutputStream(partialPath.toFile());
+ dataOut = new DataOutputStream(fileOut);
+ dataOut.writeUTF(MinimalLockingWriteAheadLog.class.getName());
+ dataOut.writeInt(getVersion());
+ dataOut.writeUTF(serde.getClass().getName());
+ dataOut.writeInt(serde.getVersion());
+ dataOut.writeLong(maxTransactionId);
+ dataOut.writeInt(records.size());
+
+ for (final T record : records) {
+ logger.trace("Checkpointing {}", record);
+ serde.serializeRecord(record, dataOut);
+ }
+
+ dataOut.writeInt(swapLocations.size());
+ for (final String swapLocation : swapLocations) {
+ dataOut.writeUTF(swapLocation);
+ }
+ } finally {
+ if (dataOut != null) {
+ try {
+ dataOut.flush();
+ fileOut.getFD().sync();
+ dataOut.close();
+ } catch (final IOException e) {
+ logger.warn("Failed to close Data Stream due to {}", e.toString(), e);
+ }
+ }
+ }
+
+ // delete the snapshot, if it exists, and rename the .partial to
+ // snapshot
+ Files.deleteIfExists(snapshotPath);
+ Files.move(partialPath, snapshotPath);
+
+ // clear all of the edit logs
+ final long partitionStart = System.nanoTime();
+ for (final Partition<T> partition : partitions) {
+ // we can call clearOld without claiming the partition because it
+ // does not change the partition's state
+ // and the only member variable it touches cannot be modified, other
+ // than when #rollover() is called.
+ // And since this method is the only one that calls #rollover() and
+ // this method is synchronized,
+ // the value of that member variable will not change. And it's
+ // volatile, so we will get the correct value.
+ partition.clearOld();
+ }
+ final long partitionEnd = System.nanoTime();
+ numberBlackListedPartitions.set(0);
+
+ final long endNanos = System.nanoTime();
+ final long millis = TimeUnit.MILLISECONDS.convert(endNanos - startNanos, TimeUnit.NANOSECONDS);
+ final long partitionMillis = TimeUnit.MILLISECONDS.convert(partitionEnd - partitionStart, TimeUnit.NANOSECONDS);
+ final long stopTheWorldMillis = TimeUnit.NANOSECONDS.toMillis(stopTheWorldNanos);
+
+ logger.info("{} checkpointed with {} Records and {} Swap Files in {} milliseconds (Stop-the-world "
+ + "time = {} milliseconds, Clear Edit Logs time = {} millis), max Transaction ID {}",
+ new Object[]{this, records.size(), swapLocations.size(), millis, stopTheWorldMillis, partitionMillis, maxTransactionId});
+
+ return records.size();
+ }
+
+ @Override
+ public void shutdown() throws IOException {
+ writeLock.lock();
+ try {
+ for (final Partition<T> partition : partitions) {
+ partition.close();
+ }
+ } finally {
+ writeLock.unlock();
+ lockChannel.close();
+ }
+ }
+
+ public int getVersion() {
+ return 1;
+ }
+
+ /**
+ * Represents a partition of this repository, which maps directly to a
+ * .journal file.
+ *
+ * All methods with the exceptions of {@link #claim()}, {@link #tryClaim()},
+ * and {@link #releaseClaim()} in this Partition MUST be called while
+ * holding the claim (via {@link #claim} or {@link #tryClaim()}).
+ *
+ * @param <S> type of record held in the partitions
+ */
+ private static class Partition<S> {
+
+ public static final String JOURNAL_EXTENSION = ".journal";
+ private static final Pattern JOURNAL_FILENAME_PATTERN = Pattern.compile("\\d+\\.journal");
+
+ private final SerDe<S> serde;
+
+ private final Path editDirectory;
+ private final int writeAheadLogVersion;
+
+ private final Lock lock = new ReentrantLock();
+ private DataOutputStream dataOut = null;
+ private FileOutputStream fileOut = null;
+ private boolean blackListed = false;
+ private boolean closed = false;
+ private DataInputStream recoveryIn;
+ private int recoveryVersion;
+ private String currentJournalFilename = "";
+
+ private static final byte TRANSACTION_CONTINUE = 1;
+ private static final byte TRANSACTION_COMMIT = 2;
+
+ private final String description;
+ private final AtomicLong maxTransactionId = new AtomicLong(-1L);
+ private final Logger logger = LoggerFactory.getLogger(MinimalLockingWriteAheadLog.class);
+
+ private final Queue<Path> recoveryFiles;
+
+ public Partition(final Path path, final SerDe<S> serde, final int partitionIndex, final int writeAheadLogVersion) throws IOException {
+ this.editDirectory = path;
+ this.serde = serde;
+
+ final File file = path.toFile();
+ if (!file.exists() && !file.mkdirs()) {
+ throw new IOException("Could not create directory " + file.getAbsolutePath());
+ }
+
+ this.recoveryFiles = new LinkedBlockingQueue<>();
+ for (final Path recoveryPath : getRecoveryPaths()) {
+ recoveryFiles.add(recoveryPath);
+ }
+
+ this.description = "Partition-" + partitionIndex;
+ this.writeAheadLogVersion = writeAheadLogVersion;
+ }
+
+ public boolean tryClaim() {
+ final boolean obtainedLock = lock.tryLock();
+ if (!obtainedLock) {
+ return false;
+ }
+
+ // Check if the partition is blacklisted. If so, unlock it and return false. Otherwise,
+ // leave it locked and return true, so that the caller will need to unlock.
+ if (blackListed) {
+ lock.unlock();
+ return false;
+ }
+
+ return true;
+ }
+
+ public void releaseClaim() {
+ lock.unlock();
+ }
+
+ public void close() {
+ final DataOutputStream out = dataOut;
+ if (out != null) {
+ try {
+ out.close();
+ } catch (final Exception e) {
+
+ }
+ }
+
+ this.closed = true;
+ this.dataOut = null;
+ }
+
+ public void blackList() {
+ lock.lock();
+ try {
+ blackListed = true;
+ } finally {
+ lock.unlock();
+ }
+ logger.debug("Blacklisted {}", this);
+ }
+
+ /**
+ * Closes resources pointing to the current journal and begins writing
+ * to a new one
+ *
+ * @throws IOException if failure to rollover
+ */
+ public void rollover() throws IOException {
+ lock.lock();
+ try {
+ final DataOutputStream out = dataOut;
+ if (out != null) {
+ out.close();
+ }
+
+ final Path editPath = getNewEditPath();
+ final FileOutputStream fos = new FileOutputStream(editPath.toFile());
+ final DataOutputStream outStream = new DataOutputStream(new BufferedOutputStream(fos));
+ outStream.writeUTF(MinimalLockingWriteAheadLog.class.getName());
+ outStream.writeInt(writeAheadLogVersion);
+ outStream.writeUTF(serde.getClass().getName());
+ outStream.writeInt(serde.getVersion());
+ outStream.flush();
+ dataOut = outStream;
+ fileOut = fos;
+
+ currentJournalFilename = editPath.toFile().getName();
+
+ blackListed = false;
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ private long getJournalIndex(final File file) {
+ final String filename = file.getName();
+ final int dotIndex = filename.indexOf(".");
+ final String number = filename.substring(0, dotIndex);
+ return Long.parseLong(number);
+ }
+
+ private Path getNewEditPath() {
+ final List<Path> recoveryPaths = getRecoveryPaths();
+ final long newIndex;
+ if (recoveryPaths == null || recoveryPaths.isEmpty()) {
+ newIndex = 1;
+ } else {
+ final long lastFileIndex = getJournalIndex(recoveryPaths.get(recoveryPaths.size() - 1).toFile());
+ newIndex = lastFileIndex + 1;
+ }
+
+ return editDirectory.resolve(newIndex + JOURNAL_EXTENSION);
+ }
+
+ private List<Path> getRecoveryPaths() {
+ final List<Path> paths = new ArrayList<>();
+
+ final File directory = editDirectory.toFile();
+ final File[] partitionFiles = directory.listFiles();
+ if (partitionFiles == null) {
+ return paths;
+ }
+
+ for (final File file : partitionFiles) {
+ // if file is a journal file but no data has yet been persisted, it may
+ // very well be a 0-byte file (the journal is not SYNC'ed to disk after
+ // a header is written out, so it may be lost). In this case, the journal
+ // is empty, so we can just skip it.
+ if (file.isDirectory() || file.length() == 0L) {
+ continue;
+ }
+
+ if (!JOURNAL_FILENAME_PATTERN.matcher(file.getName()).matches()) {
+ continue;
+ }
+
+ if (isJournalFile(file)) {
+ paths.add(file.toPath());
+ } else {
+ logger.warn("Found file {}, but could not access it, or it was not in the expected format; "
+ + "will ignore this file", file.getAbsolutePath());
+ }
+ }
+
+ // Sort journal files by the numeric portion of the filename
+ Collections.sort(paths, new Comparator<Path>() {
+ @Override
+ public int compare(final Path o1, final Path o2) {
+ if (o1 == null && o2 == null) {
+ return 0;
+ }
+ if (o1 == null) {
+ return 1;
+ }
+ if (o2 == null) {
+ return -1;
+ }
+
+ final long index1 = getJournalIndex(o1.toFile());
+ final long index2 = getJournalIndex(o2.toFile());
+ return Long.compare(index1, index2);
+ }
+ });
+
+ return paths;
+ }
+
+ void clearOld() {
+ final List<Path> oldRecoveryFiles = getRecoveryPaths();
+
+ for (final Path path : oldRecoveryFiles) {
+ final File file = path.toFile();
+ if (file.getName().equals(currentJournalFilename)) {
+ continue;
+ }
+ if (file.exists()) {
+ file.delete();
+ }
+ }
+ }
+
+ private boolean isJournalFile(final File file) {
+ final String expectedStartsWith = MinimalLockingWriteAheadLog.class.getName();
+ try {
+ try (final FileInputStream fis = new FileInputStream(file);
+ final InputStream bufferedIn = new BufferedInputStream(fis);
+ final DataInputStream in = new DataInputStream(bufferedIn)) {
+ final String waliImplClassName = in.readUTF();
+ if (!expectedStartsWith.equals(waliImplClassName)) {
+ return false;
+ }
+ }
+ } catch (final IOException e) {
+ return false;
+ }
+
+ return true;
+ }
+
+ public void update(final Collection<S> records, final long transactionId, final Map<Object, S> recordMap, final boolean forceSync)
+ throws IOException {
+ if (this.closed) {
+ throw new IllegalStateException("Partition is closed");
+ }
+
+ final DataOutputStream out = dataOut;
+ out.writeLong(transactionId);
+
+ final int numEditsToSerialize = records.size();
+ int editsSerialized = 0;
+ for (final S record : records) {
+ final Object recordId = serde.getRecordIdentifier(record);
+ final S previousVersion = recordMap.get(recordId);
+
+ serde.serializeEdit(previousVersion, record, out);
+ if (++editsSerialized < numEditsToSerialize) {
+ out.write(TRANSACTION_CONTINUE);
+ } else {
+ out.write(TRANSACTION_COMMIT);
+ }
+ }
+
+ out.flush();
+
+ if (forceSync) {
+ fileOut.getFD().sync();
+ }
+ }
+
+ private DataInputStream createDataInputStream(final Path path) throws IOException {
+ return new DataInputStream(new BufferedInputStream(Files.newInputStream(path)));
+ }
+
+ private DataInputStream getRecoveryStream() throws IOException {
+ if (recoveryIn != null && hasMoreData(recoveryIn)) {
+ return recoveryIn;
+ }
+
+ while (true) {
+ final Path nextRecoveryPath = recoveryFiles.poll();
+ if (nextRecoveryPath == null) {
+ return null;
+ }
+
+ logger.debug("{} recovering from {}", this, nextRecoveryPath);
+ recoveryIn = createDataInputStream(nextRecoveryPath);
+ if (hasMoreData(recoveryIn)) {
+ final String waliImplementationClass = recoveryIn.readUTF();
+ if (!MinimalLockingWriteAheadLog.class.getName().equals(waliImplementationClass)) {
+ continue;
+ }
+
+ final long waliVersion = recoveryIn.readInt();
+ if (waliVersion > writeAheadLogVersion) {
+ throw new IOException("Cannot recovery from file " + nextRecoveryPath + " because it was written using "
+ + "WALI version " + waliVersion + ", but the version used to restore it is only " + writeAheadLogVersion);
+ }
+
+ @SuppressWarnings("unused")
+ final String serdeClassName = recoveryIn.readUTF();
+ this.recoveryVersion = recoveryIn.readInt();
+
+ break;
+ }
+ }
+
+ return recoveryIn;
+ }
+
+ public Long getNextRecoverableTransactionId() throws IOException {
+ while (true) {
+ DataInputStream recoveryStream = getRecoveryStream();
+ if (recoveryStream == null) {
+ return null;
+ }
+
+ final long transactionId;
+ try {
+ transactionId = recoveryIn.readLong();
+ } catch (final EOFException e) {
+ continue;
+ }
+
+ this.maxTransactionId.set(transactionId);
+ return transactionId;
+ }
+ }
+
+ private boolean hasMoreData(final InputStream in) throws IOException {
+ in.mark(1);
+ final int nextByte = in.read();
+ in.reset();
+ return nextByte >= 0;
+ }
+
+ public void endRecovery() throws IOException {
+ if (recoveryIn != null) {
+ recoveryIn.close();
+ }
+
+ final Path nextRecoveryPath = this.recoveryFiles.poll();
+ if (nextRecoveryPath != null) {
+ throw new IllegalStateException("Signaled to end recovery, but there are more recovery files for Partition "
+ + "in directory " + editDirectory);
+ }
+
+ final Path newEditPath = getNewEditPath();
+
+ final FileOutputStream fos = new FileOutputStream(newEditPath.toFile());
+ final DataOutputStream outStream = new DataOutputStream(new BufferedOutputStream(fos));
+ outStream.writeUTF(MinimalLockingWriteAheadLog.class.getName());
+ outStream.writeInt(writeAheadLogVersion);
+ outStream.writeUTF(serde.getClass().getName());
+ outStream.writeInt(serde.getVersion());
+ outStream.flush();
+ dataOut = outStream;
+ fileOut = fos;
+ }
+
+ public Set<Object> recoverNextTransaction(final Map<Object, S> currentRecordMap, final Map<Object, S> updatedRecordMap, final Set<String> swapLocations) throws IOException {
+ final Set<Object> idsRemoved = new HashSet<>();
+
+ int transactionFlag;
+ do {
+ final S record = serde.deserializeEdit(recoveryIn, currentRecordMap, recoveryVersion);
+ if (logger.isDebugEnabled()) {
+ logger.debug("{} Recovering Transaction {}: {}", new Object[] { this, maxTransactionId.get(), record });
+ }
+
+ final Object recordId = serde.getRecordIdentifier(record);
+ final UpdateType updateType = serde.getUpdateType(record);
+ if (updateType == UpdateType.DELETE) {
+ updatedRecordMap.remove(recordId);
+ idsRemoved.add(recordId);
+ } else if (updateType == UpdateType.SWAP_IN) {
+ final String location = serde.getLocation(record);
+ if (location == null) {
+ logger.error("Recovered SWAP_IN record from edit log, but it did not contain a Location; skipping record");
+ } else {
+ swapLocations.remove(location);
+ updatedRecordMap.put(recordId, record);
+ idsRemoved.remove(recordId);
+ }
+ } else if (updateType == UpdateType.SWAP_OUT) {
+ final String location = serde.getLocation(record);
+ if (location == null) {
+ logger.error("Recovered SWAP_OUT record from edit log, but it did not contain a Location; skipping record");
+ } else {
+ swapLocations.add(location);
+ updatedRecordMap.remove(recordId);
+ idsRemoved.add(recordId);
+ }
+ } else {
+ updatedRecordMap.put(recordId, record);
+ idsRemoved.remove(recordId);
+ }
+
+ transactionFlag = recoveryIn.read();
+ } while (transactionFlag != TRANSACTION_COMMIT);
+
+ return idsRemoved;
+ }
+
+ /**
+ * Must be called after recovery has finished
+ *
+ * @return max recovered transaction id
+ */
+ public long getMaxRecoveredTransactionId() {
+ return maxTransactionId.get();
+ }
+
+ @Override
+ public String toString() {
+ return description;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-write-ahead-log/src/main/java/org/wali/SerDe.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-write-ahead-log/src/main/java/org/wali/SerDe.java b/nifi-commons/nifi-write-ahead-log/src/main/java/org/wali/SerDe.java
new file mode 100644
index 0000000..cc984a6
--- /dev/null
+++ b/nifi-commons/nifi-write-ahead-log/src/main/java/org/wali/SerDe.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.wali;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * A mechanism for Serializing and De-Serializing a Record of a given Type
+ *
+ * @param <T> the type of record that is to be Serialized and De-Serialized by
+ * this object
+ */
+public interface SerDe<T> {
+
+ /**
+ * <p>
+ * Serializes an Edit Record to the log via the given
+ * {@link DataOutputStream}.
+ * </p>
+ *
+ * @param previousRecordState previous state
+ * @param newRecordState new state
+ * @param out stream to write to
+ * @throws IOException if fail during write
+ */
+ void serializeEdit(T previousRecordState, T newRecordState, DataOutputStream out) throws IOException;
+
+ /**
+ * <p>
+ * Serializes a Record in a form suitable for a Snapshot via the given
+ * {@link DataOutputStream}.
+ * </p>
+ *
+ * @param record to serialize
+ * @param out to write to
+ * @throws IOException if failed to write
+ */
+ void serializeRecord(T record, DataOutputStream out) throws IOException;
+
+ /**
+ * <p>
+ * Reads an Edit Record from the given {@link DataInputStream} and merges
+ * that edit with the current version of the record, returning the new,
+ * merged version. If the Edit Record indicates that the entity was deleted,
+ * must return a Record with an UpdateType of {@link UpdateType#DELETE}.
+ * This method must never return <code>null</code>.
+ * </p>
+ *
+ * @param in to deserialize from
+ * @param currentRecordStates an unmodifiable map of Record ID's to the
+ * current state of that record
+ * @param version the version of the SerDe that was used to serialize the
+ * edit record
+ * @return deserialized record
+ * @throws IOException if failure reading
+ */
+ T deserializeEdit(DataInputStream in, Map<Object, T> currentRecordStates, int version) throws IOException;
+
+ /**
+ * <p>
+ * Reads a Record from the given {@link DataInputStream} and returns this
+ * record. If no data is available, returns <code>null</code>.
+ * </p>
+ *
+ * @param in stream to read from
+ * @param version the version of the SerDe that was used to serialize the
+ * record
+ * @return record
+ * @throws IOException failure reading
+ */
+ T deserializeRecord(DataInputStream in, int version) throws IOException;
+
+ /**
+ * Returns the unique ID for the given record
+ *
+ * @param record to obtain identifier for
+ * @return identifier of record
+ */
+ Object getRecordIdentifier(T record);
+
+ /**
+ * Returns the UpdateType for the given record
+ *
+ * @param record to retrieve update type for
+ * @return update type
+ */
+ UpdateType getUpdateType(T record);
+
+ /**
+ * Returns the external location of the given record; this is used when a
+ * record is moved away from WALI or is being re-introduced to WALI. For
+ * example, WALI can be updated with a record of type
+ * {@link UpdateType#SWAP_OUT} that indicates a Location of
+ * file://tmp/external1 and can then be re-introduced to WALI by updating
+ * WALI with a record of type {@link UpdateType#CREATE} that indicates a
+ * Location of file://tmp/external1
+ *
+ * @param record to get location of
+ * @return location
+ */
+ String getLocation(T record);
+
+ /**
+ * Returns the version that this SerDe will use when writing. This used used
+ * when serializing/deserializing the edit logs so that if the version
+ * changes, we are still able to deserialize old versions
+ *
+ * @return version
+ */
+ int getVersion();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-write-ahead-log/src/main/java/org/wali/SyncListener.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-write-ahead-log/src/main/java/org/wali/SyncListener.java b/nifi-commons/nifi-write-ahead-log/src/main/java/org/wali/SyncListener.java
new file mode 100644
index 0000000..ffb11ca
--- /dev/null
+++ b/nifi-commons/nifi-write-ahead-log/src/main/java/org/wali/SyncListener.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.wali;
+
+/**
+ * <p>
+ * Provides a callback mechanism by which applicable listeners can be notified
+ * when a WriteAheadRepository is synched (via the
+ * {@link WriteAheadRepository#sync()} method) or one of its partitions is
+ * synched via
+ * {@link WriteAheadRepository#update(java.util.Collection, boolean)} with a
+ * value of <code>true</code> for the second argument.
+ * </p>
+ *
+ * <p>
+ * It is not required that an implementation of {@link WriteAheadRepository}
+ * support this interface. Those that do generally will require that the
+ * listener be injected via the constructor.
+ * </p>
+ *
+ * <p>
+ * All implementations of this interface must be thread-safe.
+ * </p>
+ *
+ * <p>
+ * The {@link #onSync(int)} method will always be called while the associated
+ * partition is locked. The {@link #onGlobalSync()} will always be called while
+ * the entire repository is locked.
+ * </p>
+ *
+ */
+public interface SyncListener {
+
+ /**
+ * This method is called whenever a specific partition is synched via the
+ * {@link WriteAheadRepository#update(java.util.Collection, boolean)} method
+ *
+ * @param partitionIndex the index of the partition that was synched
+ */
+ void onSync(int partitionIndex);
+
+ /**
+ * This method is called whenever the entire
+ * <code>WriteAheadRepository</code> is synched via the
+ * {@link WriteAheadRepository#sync()} method.
+ */
+ void onGlobalSync();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-write-ahead-log/src/main/java/org/wali/UpdateType.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-write-ahead-log/src/main/java/org/wali/UpdateType.java b/nifi-commons/nifi-write-ahead-log/src/main/java/org/wali/UpdateType.java
new file mode 100644
index 0000000..1b039f8
--- /dev/null
+++ b/nifi-commons/nifi-write-ahead-log/src/main/java/org/wali/UpdateType.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.wali;
+
+/**
+ * <p>
+ * Enumerates the valid types of things that can cause a
+ * {@link WriteAheadRepository} to update its state</p>
+ */
+public enum UpdateType {
+
+ /**
+ * Used when a new Record has been created
+ */
+ CREATE,
+ /**
+ * Used when a Record has been updated in some way
+ */
+ UPDATE,
+ /**
+ * Used to indicate that a Record has been deleted and should be removed
+ * from the Repository
+ */
+ DELETE,
+ /**
+ * Used to indicate that a Record still exists but has been moved elsewhere,
+ * so that it is no longer maintained by the WALI instance
+ */
+ SWAP_OUT,
+ /**
+ * Used to indicate that a Record that was previously Swapped Out is now
+ * being Swapped In
+ */
+ SWAP_IN;
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-write-ahead-log/src/main/java/org/wali/WriteAheadRepository.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-write-ahead-log/src/main/java/org/wali/WriteAheadRepository.java b/nifi-commons/nifi-write-ahead-log/src/main/java/org/wali/WriteAheadRepository.java
new file mode 100644
index 0000000..7f0e828
--- /dev/null
+++ b/nifi-commons/nifi-write-ahead-log/src/main/java/org/wali/WriteAheadRepository.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.wali;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Set;
+
+/**
+ * <p>
+ * A WriteAheadRepository is used to persist state that is otherwise kept
+ * in-memory. The Repository does not provide any query capability except to
+ * allow the data to be recovered upon restart of the system.
+ * </p>
+ *
+ * <p>
+ * A WriteAheadRepository operates by writing every update to an Edit Log. On
+ * restart, the data can be recovered by replaying all of the updates that are
+ * found in the Edit Log. This can, however, eventually result in very large
+ * Edit Logs, which can both take up massive amounts of disk space and take a
+ * long time to recover. In order to prevent this, the Repository provides a
+ * Checkpointing capability. This allows the current in-memory state of the
+ * Repository to be flushed to disk and the Edit Log to be deleted, thereby
+ * compacting the amount of space required to store the Repository. After a
+ * Checkpoint is performed, modifications are again written to an Edit Log. At
+ * this point, when the system is to be restored, it is restored by first
+ * loading the Checkpointed version of the Repository and then replaying the
+ * Edit Log.
+ * </p>
+ *
+ * <p>
+ * All implementations of <code>WriteAheadRepository</code> use one or more
+ * partitions to manage their Edit Logs. An implementation may require exactly
+ * one partition or may allow many partitions.
+ * </p>
+ *
+ * @param <T> the type of Record this repository is for
+ */
+public interface WriteAheadRepository<T> {
+
+ /**
+ * <p>
+ * Updates the repository with the specified Records. The Collection must
+ * not contain multiple records with the same ID
+ * </p>
+ *
+ * @param records the records to update
+ * @param forceSync specifies whether or not the Repository forces the data
+ * to be flushed to disk. If false, the data may be stored in Operating
+ * System buffers, which improves performance but could cause loss of data
+ * if power is lost or the Operating System crashes
+ * @throws IOException if failure to update repo
+ * @throws IllegalArgumentException if multiple records within the given
+ * Collection have the same ID, as specified by {@link Record#getId()}
+ * method
+ *
+ * @return the index of the Partition that performed the update
+ */
+ int update(Collection<T> records, boolean forceSync) throws IOException;
+
+ /**
+ * <p>
+ * Recovers all records from the persisted state. This method must be called
+ * before any updates are issued to the Repository.
+ * </p>
+ *
+ * @return recovered records
+ * @throws IOException if failure to read from repo
+ * @throws IllegalStateException if any updates have been issued against
+ * this Repository before this method is invoked
+ */
+ Collection<T> recoverRecords() throws IOException;
+
+ /**
+ * <p>
+ * Recovers all External Swap locations that were persisted. If this method
+ * is to be called, it must be called AFTER {@link #recoverRecords()} and
+ * BEFORE {@link update}.
+ * </p>
+ *
+ * @return swap location
+ * @throws IOException if failure reading swap locations
+ */
+ Set<String> getRecoveredSwapLocations() throws IOException;
+
+ /**
+ * <p>
+ * Compacts the contents of the Repository so that rather than having a
+ * Snapshot and an Edit Log indicating many Updates to the Snapshot, the
+ * Snapshot is updated to contain the current state of the Repository, and
+ * the edit log is purged.
+ * </p>
+ *
+ *
+ * @return the number of records that were written to the new snapshot
+ * @throws java.io.IOException if failure during checkpoint
+ */
+ int checkpoint() throws IOException;
+
+ /**
+ * <p>
+ * Causes the repository to checkpoint and then close any open resources.
+ * </p>
+ *
+ * @throws IOException if failure to shutdown cleanly
+ */
+ void shutdown() throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-write-ahead-log/src/test/java/org/wali/DummyRecord.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-write-ahead-log/src/test/java/org/wali/DummyRecord.java b/nifi-commons/nifi-write-ahead-log/src/test/java/org/wali/DummyRecord.java
new file mode 100644
index 0000000..bf15ba7
--- /dev/null
+++ b/nifi-commons/nifi-write-ahead-log/src/test/java/org/wali/DummyRecord.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.wali;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+public class DummyRecord {
+
+ private final String id;
+ private final Map<String, String> props;
+ private final UpdateType updateType;
+
+ public DummyRecord(final String id, final UpdateType updateType) {
+ this.id = id;
+ this.props = new HashMap<>();
+ this.updateType = updateType;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public UpdateType getUpdateType() {
+ return updateType;
+ }
+
+ public DummyRecord setProperties(final Map<String, String> props) {
+ this.props.clear();
+ this.props.putAll(props);
+ return this;
+ }
+
+ public DummyRecord setProperty(final String name, final String value) {
+ this.props.put(name, value);
+ return this;
+ }
+
+ public Map<String, String> getProperties() {
+ return Collections.unmodifiableMap(this.props);
+ }
+
+ public String getProperty(final String name) {
+ return props.get(name);
+ }
+
+ @Override
+ public String toString() {
+ return "DummyRecord [id=" + id + ", props=" + props + ", updateType=" + updateType + "]";
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-write-ahead-log/src/test/java/org/wali/DummyRecordSerde.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-write-ahead-log/src/test/java/org/wali/DummyRecordSerde.java b/nifi-commons/nifi-write-ahead-log/src/test/java/org/wali/DummyRecordSerde.java
new file mode 100644
index 0000000..3a4e79f
--- /dev/null
+++ b/nifi-commons/nifi-write-ahead-log/src/test/java/org/wali/DummyRecordSerde.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.wali;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.util.Map;
+
+public class DummyRecordSerde implements SerDe<DummyRecord> {
+
+ public static final int NUM_UPDATE_TYPES = UpdateType.values().length;
+ private int throwIOEAfterNserializeEdits = -1;
+ private int throwOOMEAfterNserializeEdits = -1;
+ private int serializeEditCount = 0;
+
+ @Override
+ public void serializeEdit(final DummyRecord previousState, final DummyRecord record, final DataOutputStream out) throws IOException {
+ if (throwIOEAfterNserializeEdits >= 0 && (serializeEditCount++ >= throwIOEAfterNserializeEdits)) {
+ throw new IOException("Serialized " + (serializeEditCount - 1) + " records successfully, so now it's time to throw IOE");
+ }
+ if (throwOOMEAfterNserializeEdits >= 0 && (serializeEditCount++ >= throwOOMEAfterNserializeEdits)) {
+ throw new OutOfMemoryError("Serialized " + (serializeEditCount - 1) + " records successfully, so now it's time to throw OOME");
+ }
+
+ out.write(record.getUpdateType().ordinal());
+ out.writeUTF(record.getId());
+
+ if (record.getUpdateType() != UpdateType.DELETE) {
+ final Map<String, String> props = record.getProperties();
+ out.writeInt(props.size());
+ for (final Map.Entry<String, String> entry : props.entrySet()) {
+ out.writeUTF(entry.getKey());
+ out.writeUTF(entry.getValue());
+ }
+ }
+ }
+
+ @Override
+ public void serializeRecord(final DummyRecord record, final DataOutputStream out) throws IOException {
+ serializeEdit(null, record, out);
+ }
+
+ @Override
+ public DummyRecord deserializeRecord(final DataInputStream in, final int version) throws IOException {
+ final int index = in.read();
+ if (index < 0) {
+ throw new EOFException();
+ }
+ if (index >= NUM_UPDATE_TYPES) {
+ throw new IOException("Corrupt stream; got UpdateType value of " + index + " but there are only " + NUM_UPDATE_TYPES + " valid values");
+ }
+ final UpdateType updateType = UpdateType.values()[index];
+ final String id = in.readUTF();
+ final DummyRecord record = new DummyRecord(id, updateType);
+
+ if (record.getUpdateType() != UpdateType.DELETE) {
+ final int numProps = in.readInt();
+ for (int i = 0; i < numProps; i++) {
+ final String key = in.readUTF();
+ final String value = in.readUTF();
+ record.setProperty(key, value);
+ }
+ }
+ return record;
+ }
+
+ @Override
+ public Object getRecordIdentifier(final DummyRecord record) {
+ return record.getId();
+ }
+
+ @Override
+ public UpdateType getUpdateType(final DummyRecord record) {
+ return record.getUpdateType();
+ }
+
+ @Override
+ public DummyRecord deserializeEdit(final DataInputStream in, final Map<Object, DummyRecord> currentVersion, final int version) throws IOException {
+ return deserializeRecord(in, version);
+ }
+
+ @Override
+ public int getVersion() {
+ return 1;
+ }
+
+ public void setThrowIOEAfterNSerializeEdits(final int n) {
+ this.throwIOEAfterNserializeEdits = n;
+ }
+
+ public void setThrowOOMEAfterNSerializeEdits(final int n) {
+ this.throwOOMEAfterNserializeEdits = n;
+ }
+
+ @Override
+ public String getLocation(final DummyRecord record) {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-write-ahead-log/src/test/java/org/wali/TestMinimalLockingWriteAheadLog.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-write-ahead-log/src/test/java/org/wali/TestMinimalLockingWriteAheadLog.java b/nifi-commons/nifi-write-ahead-log/src/test/java/org/wali/TestMinimalLockingWriteAheadLog.java
new file mode 100644
index 0000000..29d2e7f
--- /dev/null
+++ b/nifi-commons/nifi-write-ahead-log/src/test/java/org/wali/TestMinimalLockingWriteAheadLog.java
@@ -0,0 +1,513 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.wali;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestMinimalLockingWriteAheadLog {
+ private static final Logger logger = LoggerFactory.getLogger(TestMinimalLockingWriteAheadLog.class);
+
+
+ @Test
+ public void testRepoDoesntContinuallyGrowOnOutOfMemoryError() throws IOException, InterruptedException {
+ final int numPartitions = 8;
+
+ final Path path = Paths.get("target/minimal-locking-repo");
+ deleteRecursively(path.toFile());
+ assertTrue(path.toFile().mkdirs());
+
+ final DummyRecordSerde serde = new DummyRecordSerde();
+ final WriteAheadRepository<DummyRecord> repo = new MinimalLockingWriteAheadLog<>(path, numPartitions, serde, null);
+ try {
+ final Collection<DummyRecord> initialRecs = repo.recoverRecords();
+ assertTrue(initialRecs.isEmpty());
+
+ serde.setThrowOOMEAfterNSerializeEdits(100);
+ for (int i = 0; i < 108; i++) {
+ try {
+ final DummyRecord record = new DummyRecord(String.valueOf(i), UpdateType.CREATE);
+ repo.update(Collections.singleton(record), false);
+ } catch (final OutOfMemoryError oome) {
+ logger.info("Received OOME on record " + i);
+ }
+ }
+
+ long expectedSize = sizeOf(path.toFile());
+ for (int i = 0; i < 1000; i++) {
+ try {
+ final DummyRecord record = new DummyRecord(String.valueOf(i), UpdateType.CREATE);
+ repo.update(Collections.singleton(record), false);
+ Assert.fail("Expected IOE but it didn't happen");
+ } catch (final IOException ioe) {
+ // will get IOException because all Partitions have been blacklisted
+ }
+ }
+
+ long newSize = sizeOf(path.toFile());
+ assertEquals(expectedSize, newSize);
+
+ try {
+ repo.checkpoint();
+ Assert.fail("Expected OOME but it didn't happen");
+ } catch (final OutOfMemoryError oome) {
+ }
+
+ expectedSize = sizeOf(path.toFile());
+
+ for (int i = 0; i < 100000; i++) {
+ try {
+ final DummyRecord record = new DummyRecord(String.valueOf(i), UpdateType.CREATE);
+ repo.update(Collections.singleton(record), false);
+ Assert.fail("Expected IOE but it didn't happen");
+ } catch (final IOException ioe) {
+ // will get IOException because all Partitions have been blacklisted
+ }
+ }
+
+ newSize = sizeOf(path.toFile());
+ assertEquals(expectedSize, newSize);
+ } finally {
+ repo.shutdown();
+ }
+ }
+
+ /**
+ * This test is intended to continually update the Write-ahead log using many threads, then
+ * stop and restore the repository to check for any corruption. There were reports of potential threading
+ * issues leading to repository corruption. This was an attempt to replicate. It should not be run as a
+ * unit test, really, but will be left, as it can be valuable to exercise the implementation
+ *
+ * @throws IOException if unable to read from/write to the write-ahead log
+ * @throws InterruptedException if a thread is interrupted
+ */
+ @Test
+ @Ignore
+ public void tryToCauseThreadingIssue() throws IOException, InterruptedException {
+ System.setProperty("org.slf4j.simpleLogger.log.org.wali", "INFO");
+
+ final int numThreads = 12;
+ final long iterationsPerThread = 1000000;
+ final int numAttempts = 1000;
+
+ final Path path = Paths.get("D:/dummy/minimal-locking-repo");
+ path.toFile().mkdirs();
+
+ final AtomicReference<WriteAheadRepository<DummyRecord>> writeRepoRef = new AtomicReference<>();
+ final AtomicBoolean checkpointing = new AtomicBoolean(false);
+
+ final Thread bgThread = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ while (true) {
+ checkpointing.set(true);
+
+ final WriteAheadRepository<DummyRecord> repo = writeRepoRef.get();
+ if (repo != null) {
+ try {
+ repo.checkpoint();
+ } catch (IOException e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+ }
+
+ checkpointing.set(false);
+
+ try {
+ TimeUnit.SECONDS.sleep(5);
+ } catch (InterruptedException e) {
+ }
+ }
+ }
+ });
+ bgThread.setDaemon(true);
+ bgThread.start();
+
+ for (int x = 0; x < numAttempts; x++) {
+ final DummyRecordSerde serde = new DummyRecordSerde();
+ final WriteAheadRepository<DummyRecord> writeRepo = new MinimalLockingWriteAheadLog<>(path, 256, serde, null);
+ final Collection<DummyRecord> writeRecords = writeRepo.recoverRecords();
+ for (final DummyRecord record : writeRecords) {
+ assertEquals("B", record.getProperty("A"));
+ }
+
+ writeRepoRef.set(writeRepo);
+
+ final Thread[] threads = new Thread[numThreads];
+ for (int i = 0; i < numThreads; i++) {
+ final Thread t = new InlineCreationInsertThread(iterationsPerThread, writeRepo);
+ t.start();
+ threads[i] = t;
+ }
+
+ for (final Thread t : threads) {
+ t.join();
+ }
+
+ writeRepoRef.set(null);
+ writeRepo.shutdown();
+
+ boolean cp = checkpointing.get();
+ while (cp) {
+ Thread.sleep(100L);
+ cp = checkpointing.get();
+ }
+
+ final WriteAheadRepository<DummyRecord> readRepo = new MinimalLockingWriteAheadLog<>(path, 256, serde, null);
+ // ensure that we are able to recover the records properly
+ final Collection<DummyRecord> readRecords = readRepo.recoverRecords();
+ for (final DummyRecord record : readRecords) {
+ assertEquals("B", record.getProperty("A"));
+ }
+ readRepo.shutdown();
+ }
+ }
+
+ @Test
+ public void testWrite() throws IOException, InterruptedException {
+ final int numPartitions = 8;
+
+ final Path path = Paths.get("target/minimal-locking-repo");
+ deleteRecursively(path.toFile());
+ assertTrue(path.toFile().mkdirs());
+
+ final DummyRecordSerde serde = new DummyRecordSerde();
+ final WriteAheadRepository<DummyRecord> repo = new MinimalLockingWriteAheadLog<>(path, numPartitions, serde, null);
+ final Collection<DummyRecord> initialRecs = repo.recoverRecords();
+ assertTrue(initialRecs.isEmpty());
+
+ final List<InsertThread> threads = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ threads.add(new InsertThread(10000, 1000000 * i, repo));
+ }
+
+ final long start = System.nanoTime();
+ for (final InsertThread thread : threads) {
+ thread.start();
+ }
+ for (final InsertThread thread : threads) {
+ thread.join();
+ }
+ final long nanos = System.nanoTime() - start;
+ final long millis = TimeUnit.MILLISECONDS.convert(nanos, TimeUnit.NANOSECONDS);
+ System.out.println("Took " + millis + " millis to insert 1,000,000 records each in its own transaction");
+ repo.shutdown();
+
+ final WriteAheadRepository<DummyRecord> recoverRepo = new MinimalLockingWriteAheadLog<>(path, numPartitions, serde, null);
+ final Collection<DummyRecord> recoveredRecords = recoverRepo.recoverRecords();
+ assertFalse(recoveredRecords.isEmpty());
+ assertEquals(100000, recoveredRecords.size());
+ for (final DummyRecord record : recoveredRecords) {
+ final Map<String, String> recoveredProps = record.getProperties();
+ assertEquals(1, recoveredProps.size());
+ assertEquals("B", recoveredProps.get("A"));
+ }
+ }
+
+ @Test
+ public void testRecoverAfterIOException() throws IOException {
+ final int numPartitions = 5;
+ final Path path = Paths.get("target/minimal-locking-repo-test-recover-after-ioe");
+ deleteRecursively(path.toFile());
+ Files.createDirectories(path);
+
+ final DummyRecordSerde serde = new DummyRecordSerde();
+ final WriteAheadRepository<DummyRecord> repo = new MinimalLockingWriteAheadLog<>(path, numPartitions, serde, null);
+ final Collection<DummyRecord> initialRecs = repo.recoverRecords();
+ assertTrue(initialRecs.isEmpty());
+
+ serde.setThrowIOEAfterNSerializeEdits(7); // serialize the 2 transactions, then the first edit of the third transaction; then throw IOException
+
+ final List<DummyRecord> firstTransaction = new ArrayList<>();
+ firstTransaction.add(new DummyRecord("1", UpdateType.CREATE));
+ firstTransaction.add(new DummyRecord("2", UpdateType.CREATE));
+ firstTransaction.add(new DummyRecord("3", UpdateType.CREATE));
+
+ final List<DummyRecord> secondTransaction = new ArrayList<>();
+ secondTransaction.add(new DummyRecord("1", UpdateType.UPDATE).setProperty("abc", "123"));
+ secondTransaction.add(new DummyRecord("2", UpdateType.UPDATE).setProperty("cba", "123"));
+ secondTransaction.add(new DummyRecord("3", UpdateType.UPDATE).setProperty("aaa", "123"));
+
+ final List<DummyRecord> thirdTransaction = new ArrayList<>();
+ thirdTransaction.add(new DummyRecord("1", UpdateType.DELETE));
+ thirdTransaction.add(new DummyRecord("2", UpdateType.DELETE));
+
+ repo.update(firstTransaction, true);
+ repo.update(secondTransaction, true);
+ try {
+ repo.update(thirdTransaction, true);
+ Assert.fail("Did not throw IOException on third transaction");
+ } catch (final IOException e) {
+ // expected behavior.
+ }
+
+ repo.shutdown();
+
+ serde.setThrowIOEAfterNSerializeEdits(-1);
+ final WriteAheadRepository<DummyRecord> recoverRepo = new MinimalLockingWriteAheadLog<>(path, numPartitions, serde, null);
+ final Collection<DummyRecord> recoveredRecords = recoverRepo.recoverRecords();
+ assertFalse(recoveredRecords.isEmpty());
+ assertEquals(3, recoveredRecords.size());
+
+ boolean record1 = false, record2 = false, record3 = false;
+ for (final DummyRecord record : recoveredRecords) {
+ switch (record.getId()) {
+ case "1":
+ record1 = true;
+ assertEquals("123", record.getProperty("abc"));
+ break;
+ case "2":
+ record2 = true;
+ assertEquals("123", record.getProperty("cba"));
+ break;
+ case "3":
+ record3 = true;
+ assertEquals("123", record.getProperty("aaa"));
+ break;
+ }
+ }
+
+ assertTrue(record1);
+ assertTrue(record2);
+ assertTrue(record3);
+ }
+
+ @Test
+ public void testCannotModifyLogAfterAllAreBlackListed() throws IOException {
+ final int numPartitions = 5;
+ final Path path = Paths.get("target/minimal-locking-repo-test-cannot-modify-after-all-blacklisted");
+ deleteRecursively(path.toFile());
+ Files.createDirectories(path);
+
+ final DummyRecordSerde serde = new DummyRecordSerde();
+ final WriteAheadRepository<DummyRecord> repo = new MinimalLockingWriteAheadLog<>(path, numPartitions, serde, null);
+ final Collection<DummyRecord> initialRecs = repo.recoverRecords();
+ assertTrue(initialRecs.isEmpty());
+
+ serde.setThrowIOEAfterNSerializeEdits(3); // serialize the first transaction, then fail on all subsequent transactions
+
+ final List<DummyRecord> firstTransaction = new ArrayList<>();
+ firstTransaction.add(new DummyRecord("1", UpdateType.CREATE));
+ firstTransaction.add(new DummyRecord("2", UpdateType.CREATE));
+ firstTransaction.add(new DummyRecord("3", UpdateType.CREATE));
+
+ final List<DummyRecord> secondTransaction = new ArrayList<>();
+ secondTransaction.add(new DummyRecord("1", UpdateType.UPDATE).setProperty("abc", "123"));
+ secondTransaction.add(new DummyRecord("2", UpdateType.UPDATE).setProperty("cba", "123"));
+ secondTransaction.add(new DummyRecord("3", UpdateType.UPDATE).setProperty("aaa", "123"));
+
+ final List<DummyRecord> thirdTransaction = new ArrayList<>();
+ thirdTransaction.add(new DummyRecord("1", UpdateType.DELETE));
+ thirdTransaction.add(new DummyRecord("2", UpdateType.DELETE));
+
+ repo.update(firstTransaction, true);
+
+ try {
+ repo.update(secondTransaction, true);
+ Assert.fail("Did not throw IOException on second transaction");
+ } catch (final IOException e) {
+ // expected behavior.
+ }
+
+ for (int i = 0; i < 4; i++) {
+ try {
+ repo.update(thirdTransaction, true);
+ Assert.fail("Did not throw IOException on third transaction");
+ } catch (final IOException e) {
+ // expected behavior.
+ }
+ }
+
+ serde.setThrowIOEAfterNSerializeEdits(-1);
+ final List<DummyRecord> fourthTransaction = new ArrayList<>();
+ fourthTransaction.add(new DummyRecord("1", UpdateType.DELETE));
+
+ try {
+ repo.update(fourthTransaction, true);
+ Assert.fail("Successfully updated repo for 4th transaction");
+ } catch (final IOException e) {
+ // expected behavior
+ assertTrue(e.getMessage().contains("All Partitions have been blacklisted"));
+ }
+
+ repo.shutdown();
+ serde.setThrowIOEAfterNSerializeEdits(-1);
+
+ final WriteAheadRepository<DummyRecord> recoverRepo = new MinimalLockingWriteAheadLog<>(path, numPartitions, serde, null);
+ final Collection<DummyRecord> recoveredRecords = recoverRepo.recoverRecords();
+ assertFalse(recoveredRecords.isEmpty());
+ assertEquals(3, recoveredRecords.size());
+ }
+
+ @Test
+ public void testStriping() throws IOException {
+ final int numPartitions = 6;
+ final Path path = Paths.get("target/minimal-locking-repo-striped");
+ deleteRecursively(path.toFile());
+ Files.createDirectories(path);
+
+ final SortedSet<Path> paths = new TreeSet<>();
+ paths.add(path.resolve("stripe-1"));
+ paths.add(path.resolve("stripe-2"));
+
+ final DummyRecordSerde serde = new DummyRecordSerde();
+ final WriteAheadRepository<DummyRecord> repo = new MinimalLockingWriteAheadLog<>(paths, numPartitions, serde, null);
+ final Collection<DummyRecord> initialRecs = repo.recoverRecords();
+ assertTrue(initialRecs.isEmpty());
+
+ final InsertThread inserter = new InsertThread(100000, 0, repo);
+ inserter.run();
+
+ for (final Path partitionPath : paths) {
+ final File[] files = partitionPath.toFile().listFiles(new FileFilter() {
+ @Override
+ public boolean accept(File pathname) {
+ return pathname.getName().startsWith("partition");
+ }
+ });
+ assertEquals(3, files.length);
+
+ for (final File file : files) {
+ final File[] journalFiles = file.listFiles();
+ assertEquals(1, journalFiles.length);
+ }
+ }
+
+ repo.checkpoint();
+
+ }
+
+ private static class InsertThread extends Thread {
+
+ private final List<List<DummyRecord>> records;
+ private final WriteAheadRepository<DummyRecord> repo;
+
+ public InsertThread(final int numInsertions, final int startIndex, final WriteAheadRepository<DummyRecord> repo) {
+ records = new ArrayList<>();
+ for (int i = 0; i < numInsertions; i++) {
+ final DummyRecord record = new DummyRecord(String.valueOf(i + startIndex), UpdateType.CREATE);
+ record.setProperty("A", "B");
+ final List<DummyRecord> list = new ArrayList<>();
+ list.add(record);
+ records.add(list);
+ }
+ this.repo = repo;
+ }
+
+ @Override
+ public void run() {
+ try {
+ int counter = 0;
+ for (final List<DummyRecord> list : records) {
+ final boolean forceSync = (++counter == records.size());
+ repo.update(list, forceSync);
+ }
+ } catch (IOException e) {
+ Assert.fail("Failed to update: " + e.toString());
+ e.printStackTrace();
+ }
+ }
+ }
+
+
+ private static class InlineCreationInsertThread extends Thread {
+ private final long iterations;
+ private final WriteAheadRepository<DummyRecord> repo;
+
+ public InlineCreationInsertThread(final long numInsertions, final WriteAheadRepository<DummyRecord> repo) {
+ this.iterations = numInsertions;
+ this.repo = repo;
+ }
+
+ @Override
+ public void run() {
+ final List<DummyRecord> list = new ArrayList<>(1);
+ list.add(null);
+ final UpdateType[] updateTypes = new UpdateType[] { UpdateType.CREATE, UpdateType.DELETE, UpdateType.UPDATE };
+ final Random random = new Random();
+
+ for (long i = 0; i < iterations; i++) {
+ final int updateTypeIndex = random.nextInt(updateTypes.length);
+ final UpdateType updateType = updateTypes[updateTypeIndex];
+
+ final DummyRecord record = new DummyRecord(String.valueOf(i), updateType);
+ record.setProperty("A", "B");
+ list.set(0, record);
+
+ try {
+ repo.update(list, false);
+ } catch (final Throwable t) {
+ t.printStackTrace();
+ }
+ }
+ }
+ }
+
+ private void deleteRecursively(final File file) {
+ final File[] children = file.listFiles();
+ if (children != null) {
+ for (final File child : children) {
+ deleteRecursively(child);
+ }
+ }
+
+ file.delete();
+ }
+
+ private long sizeOf(final File file) {
+ long size = 0L;
+ if (file.isDirectory()) {
+ final File[] children = file.listFiles();
+ if (children != null) {
+ for (final File child : children) {
+ size += sizeOf(child);
+ }
+ }
+ }
+
+ size += file.length();
+
+ return size;
+ }
+
+}
[10/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-mock/src/main/java/org/apache/nifi/util/MockValidationContext.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/MockValidationContext.java b/nifi-mock/src/main/java/org/apache/nifi/util/MockValidationContext.java
new file mode 100644
index 0000000..d73a09b
--- /dev/null
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/MockValidationContext.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.nifi.attribute.expression.language.Query;
+import org.apache.nifi.attribute.expression.language.Query.Range;
+import org.apache.nifi.attribute.expression.language.StandardExpressionLanguageCompiler;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.PropertyValue;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.controller.ControllerService;
+import org.apache.nifi.controller.ControllerServiceLookup;
+import org.apache.nifi.expression.ExpressionLanguageCompiler;
+
+public class MockValidationContext implements ValidationContext, ControllerServiceLookup {
+
+ private final MockProcessContext context;
+ private final Map<String, Boolean> expressionLanguageSupported;
+
+ public MockValidationContext(final MockProcessContext processContext) {
+ this.context = processContext;
+
+ final Map<PropertyDescriptor, String> properties = processContext.getProperties();
+ expressionLanguageSupported = new HashMap<>(properties.size());
+ for (final PropertyDescriptor descriptor : properties.keySet()) {
+ expressionLanguageSupported.put(descriptor.getName(), descriptor.isExpressionLanguageSupported());
+ }
+ }
+
+ @Override
+ public ControllerService getControllerService(final String identifier) {
+ return context.getControllerService(identifier);
+ }
+
+ @Override
+ public PropertyValue newPropertyValue(final String rawValue) {
+ return new MockPropertyValue(rawValue, this);
+ }
+
+ @Override
+ public ExpressionLanguageCompiler newExpressionLanguageCompiler() {
+ return new StandardExpressionLanguageCompiler();
+ }
+
+ @Override
+ public ValidationContext getControllerServiceValidationContext(final ControllerService controllerService) {
+ final MockProcessContext serviceProcessContext = new MockProcessContext(controllerService, context);
+ return new MockValidationContext(serviceProcessContext);
+ }
+
+ @Override
+ public PropertyValue getProperty(final PropertyDescriptor property) {
+ return context.getProperty(property);
+ }
+
+ @Override
+ public Map<PropertyDescriptor, String> getProperties() {
+ return context.getProperties();
+ }
+
+ @Override
+ public String getAnnotationData() {
+ return context.getAnnotationData();
+ }
+
+ @Override
+ public Set<String> getControllerServiceIdentifiers(Class<? extends ControllerService> serviceType) {
+ return context.getControllerServiceIdentifiers(serviceType);
+ }
+
+ @Override
+ public ControllerServiceLookup getControllerServiceLookup() {
+ return this;
+ }
+
+ @Override
+ public boolean isControllerServiceEnabled(final String serviceIdentifier) {
+ return context.isControllerServiceEnabled(serviceIdentifier);
+ }
+
+ @Override
+ public boolean isControllerServiceEnabled(final ControllerService service) {
+ return context.isControllerServiceEnabled(service);
+ }
+
+ @Override
+ public String getControllerServiceName(final String serviceIdentifier) {
+ final ControllerServiceConfiguration configuration = context.getConfiguration(serviceIdentifier);
+ return configuration == null ? null : serviceIdentifier;
+ }
+
+ @Override
+ public boolean isValidationRequired(final ControllerService service) {
+ return true;
+ }
+
+ @Override
+ public boolean isControllerServiceEnabling(String serviceIdentifier) {
+ return context.isControllerServiceEnabling(serviceIdentifier);
+ }
+
+ public boolean isExpressionLanguagePresent(final String value) {
+ if (value == null) {
+ return false;
+ }
+
+ final List<Range> elRanges = Query.extractExpressionRanges(value);
+ return (elRanges != null && !elRanges.isEmpty());
+ }
+
+ @Override
+ public boolean isExpressionLanguageSupported(final String propertyName) {
+ final Boolean supported = expressionLanguageSupported.get(propertyName);
+ return Boolean.TRUE.equals(supported);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-mock/src/main/java/org/apache/nifi/util/ReflectionUtils.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/ReflectionUtils.java b/nifi-mock/src/main/java/org/apache/nifi/util/ReflectionUtils.java
new file mode 100644
index 0000000..940eeea
--- /dev/null
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/ReflectionUtils.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import java.lang.annotation.Annotation;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ReflectionUtils {
+
+ private final static Logger LOG = LoggerFactory.getLogger(ReflectionUtils.class);
+
+ /**
+ * Invokes all methods on the given instance that have been annotated with
+ * the given Annotation. If the signature of the method that is defined in
+ * <code>instance</code> uses 1 or more parameters, those parameters must be
+ * specified by the <code>args</code> parameter. However, if more arguments
+ * are supplied by the <code>args</code> parameter than needed, the extra
+ * arguments will be ignored.
+ *
+ * @param annotation the annotation to look for
+ * @param instance to invoke a method of
+ * @param args to supply in a method call
+ * @throws InvocationTargetException ite
+ * @throws IllegalArgumentException iae
+ * @throws IllegalAccessException if not allowed to invoke that method
+ */
+ public static void invokeMethodsWithAnnotation(final Class<? extends Annotation> annotation, final Object instance, final Object... args)
+ throws IllegalAccessException, IllegalArgumentException, InvocationTargetException {
+ for (final Method method : instance.getClass().getMethods()) {
+ if (method.isAnnotationPresent(annotation)) {
+ final boolean isAccessible = method.isAccessible();
+ method.setAccessible(true);
+
+ try {
+ final Class<?>[] argumentTypes = method.getParameterTypes();
+ if (argumentTypes.length > args.length) {
+ throw new IllegalArgumentException(String.format("Unable to invoke method %1$s on %2$s because method expects %3$s parameters but only %4$s were given",
+ method.getName(), instance, argumentTypes.length, args.length));
+ }
+
+ for (int i = 0; i < argumentTypes.length; i++) {
+ final Class<?> argType = argumentTypes[i];
+ if (!argType.isAssignableFrom(args[i].getClass())) {
+ throw new IllegalArgumentException(String.format(
+ "Unable to invoke method %1$s on %2$s because method parameter %3$s is expected to be of type %4$s but argument passed was of type %5$s",
+ method.getName(), instance, i, argType, args[i].getClass()));
+ }
+ }
+
+ if (argumentTypes.length == args.length) {
+ method.invoke(instance, args);
+ } else {
+ final Object[] argsToPass = new Object[argumentTypes.length];
+ for (int i = 0; i < argsToPass.length; i++) {
+ argsToPass[i] = args[i];
+ }
+
+ method.invoke(instance, argsToPass);
+ }
+ } finally {
+ if (!isAccessible) {
+ method.setAccessible(false);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Invokes all methods on the given instance that have been annotated with
+ * the given Annotation. If the signature of the method that is defined in
+ * <code>instance</code> uses 1 or more parameters, those parameters must be
+ * specified by the <code>args</code> parameter. However, if more arguments
+ * are supplied by the <code>args</code> parameter than needed, the extra
+ * arguments will be ignored.
+ *
+ * @param annotation the annotation to look for
+ * @param instance to invoke a method of
+ * @param args to supply in a method call
+ * @return <code>true</code> if all appropriate methods were invoked and
+ * returned without throwing an Exception, <code>false</code> if one of the
+ * methods threw an Exception or could not be invoked; if <code>false</code>
+ * is returned, an error will have been logged.
+ */
+ public static boolean quietlyInvokeMethodsWithAnnotation(final Class<? extends Annotation> annotation, final Object instance, final Object... args) {
+ for (final Method method : instance.getClass().getMethods()) {
+ if (method.isAnnotationPresent(annotation)) {
+ final boolean isAccessible = method.isAccessible();
+ method.setAccessible(true);
+
+ try {
+ final Class<?>[] argumentTypes = method.getParameterTypes();
+ if (argumentTypes.length > args.length) {
+ LOG.error("Unable to invoke method {} on {} because method expects {} parameters but only {} were given",
+ new Object[]{method.getName(), instance, argumentTypes.length, args.length});
+ return false;
+ }
+
+ for (int i = 0; i < argumentTypes.length; i++) {
+ final Class<?> argType = argumentTypes[i];
+ if (!argType.isAssignableFrom(args[i].getClass())) {
+ LOG.error("Unable to invoke method {} on {} because method parameter {} is expected to be of type {} but argument passed was of type {}",
+ new Object[]{method.getName(), instance, i, argType, args[i].getClass()});
+ return false;
+ }
+ }
+
+ try {
+ if (argumentTypes.length == args.length) {
+ method.invoke(instance, args);
+ } else {
+ final Object[] argsToPass = new Object[argumentTypes.length];
+ for (int i = 0; i < argsToPass.length; i++) {
+ argsToPass[i] = args[i];
+ }
+
+ method.invoke(instance, argsToPass);
+ }
+ } catch (final IllegalAccessException | IllegalArgumentException | InvocationTargetException t) {
+ LOG.error("Unable to invoke method {} on {} due to {}", new Object[]{method.getName(), instance, t});
+ LOG.error("", t);
+ return false;
+ }
+ } finally {
+ if (!isAccessible) {
+ method.setAccessible(false);
+ }
+ }
+ }
+ }
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-mock/src/main/java/org/apache/nifi/util/SharedSessionState.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/SharedSessionState.java b/nifi-mock/src/main/java/org/apache/nifi/util/SharedSessionState.java
new file mode 100644
index 0000000..994735b
--- /dev/null
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/SharedSessionState.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.nifi.processor.Processor;
+import org.apache.nifi.provenance.ProvenanceEventRecord;
+import org.apache.nifi.provenance.ProvenanceReporter;
+
+public class SharedSessionState {
+
+ private final MockFlowFileQueue flowFileQueue;
+ private final ProvenanceReporter provenanceReporter;
+ @SuppressWarnings("unused")
+ private final Processor processor;
+ private final AtomicLong flowFileIdGenerator;
+ private final ConcurrentMap<String, AtomicLong> counterMap = new ConcurrentHashMap<>();
+ private final Set<ProvenanceEventRecord> events = new LinkedHashSet<>();
+
+ public SharedSessionState(final Processor processor, final AtomicLong flowFileIdGenerator) {
+ flowFileQueue = new MockFlowFileQueue();
+ provenanceReporter = new MockProvenanceReporter(null, this, UUID.randomUUID().toString(), "N/A");
+ this.flowFileIdGenerator = flowFileIdGenerator;
+ this.processor = processor;
+ }
+
+ void addProvenanceEvents(final Collection<ProvenanceEventRecord> events) {
+ this.events.addAll(events);
+ }
+
+ void clearProvenanceEvents() {
+ this.events.clear();
+ }
+
+ public List<ProvenanceEventRecord> getProvenanceEvents() {
+ return new ArrayList<>(this.events);
+ }
+
+ public MockFlowFileQueue getFlowFileQueue() {
+ return flowFileQueue;
+ }
+
+ public ProvenanceReporter getProvenanceReporter() {
+ return provenanceReporter;
+ }
+
+ public long nextFlowFileId() {
+ return flowFileIdGenerator.getAndIncrement();
+ }
+
+ public void adjustCounter(final String name, final long delta) {
+ AtomicLong counter = counterMap.get(name);
+ if (counter == null) {
+ counter = new AtomicLong(0L);
+ final AtomicLong existingCounter = counterMap.putIfAbsent(name, counter);
+ if (existingCounter != null) {
+ counter = existingCounter;
+ }
+ }
+
+ counter.addAndGet(delta);
+ }
+
+ public Long getCounterValue(final String name) {
+ final AtomicLong counterValue = counterMap.get(name);
+ return counterValue == null ? null : counterValue.get();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-mock/src/main/java/org/apache/nifi/util/SingleSessionFactory.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/SingleSessionFactory.java b/nifi-mock/src/main/java/org/apache/nifi/util/SingleSessionFactory.java
new file mode 100644
index 0000000..ef518dc
--- /dev/null
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/SingleSessionFactory.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.ProcessSessionFactory;
+
+public class SingleSessionFactory implements ProcessSessionFactory {
+
+ private final MockProcessSession session;
+
+ public SingleSessionFactory(final MockProcessSession session) {
+ this.session = session;
+ }
+
+ @Override
+ public ProcessSession createSession() {
+ return session;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-mock/src/main/java/org/apache/nifi/util/StandardProcessorTestRunner.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/StandardProcessorTestRunner.java b/nifi-mock/src/main/java/org/apache/nifi/util/StandardProcessorTestRunner.java
new file mode 100644
index 0000000..048e2b9
--- /dev/null
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/StandardProcessorTestRunner.java
@@ -0,0 +1,728 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import static java.util.Objects.requireNonNull;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.annotation.Annotation;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.nifi.annotation.behavior.TriggerSerially;
+import org.apache.nifi.annotation.lifecycle.OnAdded;
+import org.apache.nifi.annotation.lifecycle.OnDisabled;
+import org.apache.nifi.annotation.lifecycle.OnEnabled;
+import org.apache.nifi.annotation.lifecycle.OnRemoved;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.annotation.lifecycle.OnShutdown;
+import org.apache.nifi.annotation.lifecycle.OnStopped;
+import org.apache.nifi.annotation.lifecycle.OnUnscheduled;
+import org.apache.nifi.components.AllowableValue;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.controller.ConfigurationContext;
+import org.apache.nifi.controller.ControllerService;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.ProcessSessionFactory;
+import org.apache.nifi.processor.Processor;
+import org.apache.nifi.processor.QueueSize;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.provenance.ProvenanceEventRecord;
+import org.apache.nifi.provenance.ProvenanceReporter;
+import org.apache.nifi.reporting.InitializationException;
+import org.junit.Assert;
+
+public class StandardProcessorTestRunner implements TestRunner {
+
+ private final Processor processor;
+ private final MockProcessContext context;
+ private final MockFlowFileQueue flowFileQueue;
+ private final MockSessionFactory sessionFactory;
+ private final SharedSessionState sharedState;
+ private final AtomicLong idGenerator;
+ private final boolean triggerSerially;
+
+ private int numThreads = 1;
+ private final AtomicInteger invocations = new AtomicInteger(0);
+
+ private static final Set<Class<? extends Annotation>> deprecatedTypeAnnotations = new HashSet<>();
+ private static final Set<Class<? extends Annotation>> deprecatedMethodAnnotations = new HashSet<>();
+
+ static {
+ // do this in a separate method, just so that we can add a @SuppressWarnings annotation
+ // because we want to indicate explicitly that we know that we are using deprecated
+ // classes here.
+ populateDeprecatedMethods();
+ }
+
+ StandardProcessorTestRunner(final Processor processor) {
+ this.processor = processor;
+ this.idGenerator = new AtomicLong(0L);
+ this.sharedState = new SharedSessionState(processor, idGenerator);
+ this.flowFileQueue = sharedState.getFlowFileQueue();
+ this.sessionFactory = new MockSessionFactory(sharedState, processor);
+ this.context = new MockProcessContext(processor);
+
+ detectDeprecatedAnnotations(processor);
+
+ final MockProcessorInitializationContext mockInitContext = new MockProcessorInitializationContext(processor, context);
+ processor.initialize(mockInitContext);
+
+ try {
+ ReflectionUtils.invokeMethodsWithAnnotation(OnAdded.class, processor);
+ } catch (final Exception e) {
+ Assert.fail("Could not invoke methods annotated with @OnAdded annotation due to: " + e);
+ }
+
+ triggerSerially = null != processor.getClass().getAnnotation(TriggerSerially.class);
+ }
+
+ @SuppressWarnings("deprecation")
+ private static void populateDeprecatedMethods() {
+ deprecatedTypeAnnotations.add(org.apache.nifi.processor.annotation.CapabilityDescription.class);
+ deprecatedTypeAnnotations.add(org.apache.nifi.processor.annotation.EventDriven.class);
+ deprecatedTypeAnnotations.add(org.apache.nifi.processor.annotation.SideEffectFree.class);
+ deprecatedTypeAnnotations.add(org.apache.nifi.processor.annotation.SupportsBatching.class);
+ deprecatedTypeAnnotations.add(org.apache.nifi.processor.annotation.Tags.class);
+ deprecatedTypeAnnotations.add(org.apache.nifi.processor.annotation.TriggerWhenEmpty.class);
+ deprecatedTypeAnnotations.add(org.apache.nifi.processor.annotation.TriggerWhenAnyDestinationAvailable.class);
+ deprecatedTypeAnnotations.add(org.apache.nifi.processor.annotation.TriggerSerially.class);
+
+ deprecatedMethodAnnotations.add(org.apache.nifi.processor.annotation.OnRemoved.class);
+ deprecatedMethodAnnotations.add(org.apache.nifi.processor.annotation.OnAdded.class);
+ deprecatedMethodAnnotations.add(org.apache.nifi.processor.annotation.OnScheduled.class);
+ deprecatedMethodAnnotations.add(org.apache.nifi.processor.annotation.OnShutdown.class);
+ deprecatedMethodAnnotations.add(org.apache.nifi.processor.annotation.OnStopped.class);
+ deprecatedMethodAnnotations.add(org.apache.nifi.processor.annotation.OnUnscheduled.class);
+ }
+
+ private static void detectDeprecatedAnnotations(final Processor processor) {
+ for (final Class<? extends Annotation> annotationClass : deprecatedTypeAnnotations) {
+ if (processor.getClass().isAnnotationPresent(annotationClass)) {
+ Assert.fail("Processor is using deprecated Annotation " + annotationClass.getCanonicalName());
+ }
+ }
+
+ for (final Class<? extends Annotation> annotationClass : deprecatedMethodAnnotations) {
+ for (final Method method : processor.getClass().getMethods()) {
+ if (method.isAnnotationPresent(annotationClass)) {
+ Assert.fail("Processor is using deprecated Annotation " + annotationClass.getCanonicalName() + " for method " + method);
+ }
+ }
+ }
+
+ }
+
+ @Override
+ public void setValidateExpressionUsage(final boolean validate) {
+ context.setValidateExpressionUsage(validate);
+ }
+
+ @Override
+ public Processor getProcessor() {
+ return processor;
+ }
+
+ @Override
+ public MockProcessContext getProcessContext() {
+ return context;
+ }
+
+ @Override
+ public void run() {
+ run(1);
+ }
+
+ @Override
+ public void run(int iterations) {
+ run(iterations, true);
+ }
+
+ @Override
+ public void run(final int iterations, final boolean stopOnFinish) {
+ run(iterations, stopOnFinish, true);
+ }
+
+ @Override
+ public void run(final int iterations, final boolean stopOnFinish, final boolean initialize) {
+ run(iterations, stopOnFinish, initialize, 5000);
+ }
+
+ @Override
+ public void run(final int iterations, final boolean stopOnFinish, final boolean initialize, final long runWait) {
+ if (iterations < 1) {
+ throw new IllegalArgumentException();
+ }
+
+ context.assertValid();
+ context.enableExpressionValidation();
+ try {
+ if (initialize) {
+ try {
+ ReflectionUtils.invokeMethodsWithAnnotation(OnScheduled.class, processor, context);
+ } catch (final Exception e) {
+ e.printStackTrace();
+ Assert.fail("Could not invoke methods annotated with @OnScheduled annotation due to: " + e);
+ }
+ }
+
+ final ExecutorService executorService = Executors.newFixedThreadPool(numThreads);
+ @SuppressWarnings("unchecked")
+ final Future<Throwable>[] futures = new Future[iterations];
+ for (int i = 0; i < iterations; i++) {
+ final Future<Throwable> future = executorService.submit(new RunProcessor());
+ futures[i] = future;
+ }
+
+ executorService.shutdown();
+ try {
+ executorService.awaitTermination(runWait, TimeUnit.MILLISECONDS);
+ } catch (final InterruptedException e1) {
+ }
+
+ int finishedCount = 0;
+ boolean unscheduledRun = false;
+ for (final Future<Throwable> future : futures) {
+ try {
+ final Throwable thrown = future.get(); // wait for the result
+ if (thrown != null) {
+ throw new AssertionError(thrown);
+ }
+
+ if (++finishedCount == 1) {
+ unscheduledRun = true;
+ try {
+ ReflectionUtils.invokeMethodsWithAnnotation(OnUnscheduled.class, processor, context);
+ } catch (final Exception e) {
+ Assert.fail("Could not invoke methods annotated with @OnUnscheduled annotation due to: " + e);
+ }
+ }
+ } catch (final Exception e) {
+ }
+ }
+
+ if (!unscheduledRun) {
+ try {
+ ReflectionUtils.invokeMethodsWithAnnotation(OnUnscheduled.class, processor, context);
+ } catch (final Exception e) {
+ Assert.fail("Could not invoke methods annotated with @OnUnscheduled annotation due to: " + e);
+ }
+ }
+
+ if (stopOnFinish) {
+ try {
+ ReflectionUtils.invokeMethodsWithAnnotation(OnStopped.class, processor);
+ } catch (final Exception e) {
+ Assert.fail("Could not invoke methods annotated with @OnStopped annotation due to: " + e);
+ }
+ }
+ } finally {
+ context.disableExpressionValidation();
+ }
+ }
+
+ @Override
+ public void shutdown() {
+ try {
+ ReflectionUtils.invokeMethodsWithAnnotation(OnShutdown.class, processor);
+ } catch (final Exception e) {
+ Assert.fail("Could not invoke methods annotated with @OnShutdown annotation due to: " + e);
+ }
+ }
+
+ private class RunProcessor implements Callable<Throwable> {
+
+ @Override
+ public Throwable call() throws Exception {
+ invocations.incrementAndGet();
+ try {
+ processor.onTrigger(context, sessionFactory);
+ } catch (final Throwable t) {
+ return t;
+ }
+
+ return null;
+ }
+ }
+
+ @Override
+ public ProcessSessionFactory getProcessSessionFactory() {
+ return sessionFactory;
+ }
+
+ @Override
+ public void assertAllFlowFilesTransferred(final String relationship) {
+ for (final MockProcessSession session : sessionFactory.getCreatedSessions()) {
+ session.assertAllFlowFilesTransferred(relationship);
+ }
+ }
+
+ @Override
+ public void assertAllFlowFilesTransferred(final Relationship relationship) {
+ for (final MockProcessSession session : sessionFactory.getCreatedSessions()) {
+ session.assertAllFlowFilesTransferred(relationship);
+ }
+ }
+
+ @Override
+ public void assertAllFlowFilesTransferred(final String relationship, final int count) {
+ assertAllFlowFilesTransferred(relationship);
+ assertTransferCount(relationship, count);
+ }
+
+ @Override
+ public void assertAllFlowFilesTransferred(final Relationship relationship, final int count) {
+ assertAllFlowFilesTransferred(relationship);
+ assertTransferCount(relationship, count);
+ }
+
+ @Override
+ public void assertTransferCount(final Relationship relationship, final int count) {
+ Assert.assertEquals(count, getFlowFilesForRelationship(relationship).size());
+ }
+
+ @Override
+ public void assertTransferCount(final String relationship, final int count) {
+ Assert.assertEquals(count, getFlowFilesForRelationship(relationship).size());
+ }
+
+ @Override
+ public void assertValid() {
+ context.assertValid();
+ }
+
+ @Override
+ public void assertNotValid() {
+ Assert.assertFalse("Processor appears to be valid but expected it to be invalid", context.isValid());
+ }
+
+ @Override
+ public boolean isQueueEmpty() {
+ return flowFileQueue.isEmpty();
+ }
+
+ @Override
+ public void assertQueueEmpty() {
+ Assert.assertTrue(flowFileQueue.isEmpty());
+ }
+
+ @Override
+ public void assertQueueNotEmpty() {
+ Assert.assertFalse(flowFileQueue.isEmpty());
+ }
+
+ @Override
+ public void clearTransferState() {
+ for (final MockProcessSession session : sessionFactory.getCreatedSessions()) {
+ session.clearTransferState();
+ }
+ }
+
+ @Override
+ public void enqueue(final FlowFile... flowFiles) {
+ for (final FlowFile flowFile : flowFiles) {
+ flowFileQueue.offer((MockFlowFile) flowFile);
+ }
+ }
+
+ @Override
+ public void enqueue(final Path path) throws IOException {
+ enqueue(path, new HashMap<String, String>());
+ }
+
+ @Override
+ public void enqueue(final Path path, final Map<String, String> attributes) throws IOException {
+ final Map<String, String> modifiedAttributes = new HashMap<>(attributes);
+ if (!modifiedAttributes.containsKey(CoreAttributes.FILENAME.key())) {
+ modifiedAttributes.put(CoreAttributes.FILENAME.key(), path.toFile().getName());
+ }
+ try (final InputStream in = Files.newInputStream(path)) {
+ enqueue(in, modifiedAttributes);
+ }
+ }
+
+ @Override
+ public void enqueue(final byte[] data) {
+ enqueue(data, new HashMap<String, String>());
+ }
+
+ @Override
+ public void enqueue(final byte[] data, final Map<String, String> attributes) {
+ enqueue(new ByteArrayInputStream(data), attributes);
+ }
+
+ @Override
+ public void enqueue(final InputStream data) {
+ enqueue(data, new HashMap<String, String>());
+ }
+
+ @Override
+ public void enqueue(final InputStream data, final Map<String, String> attributes) {
+ final MockProcessSession session = new MockProcessSession(new SharedSessionState(processor, idGenerator), processor);
+ MockFlowFile flowFile = session.create();
+ flowFile = session.importFrom(data, flowFile);
+ flowFile = session.putAllAttributes(flowFile, attributes);
+ enqueue(flowFile);
+ }
+
+ @Override
+ public byte[] getContentAsByteArray(final MockFlowFile flowFile) {
+ return flowFile.getData();
+ }
+
+ @Override
+ public List<MockFlowFile> getFlowFilesForRelationship(final String relationship) {
+ final Relationship rel = new Relationship.Builder().name(relationship).build();
+ return getFlowFilesForRelationship(rel);
+ }
+
+ @Override
+ public List<MockFlowFile> getFlowFilesForRelationship(final Relationship relationship) {
+ final List<MockFlowFile> flowFiles = new ArrayList<>();
+ for (final MockProcessSession session : sessionFactory.getCreatedSessions()) {
+ flowFiles.addAll(session.getFlowFilesForRelationship(relationship));
+ }
+
+ Collections.sort(flowFiles, new Comparator<MockFlowFile>() {
+ @Override
+ public int compare(final MockFlowFile o1, final MockFlowFile o2) {
+ return Long.compare(o1.getCreationTime(), o2.getCreationTime());
+ }
+ });
+
+ return flowFiles;
+ }
+
+ /**
+ * @deprecated The ProvenanceReporter should not be accessed through the test runner, as it does not expose the events that were emitted.
+ */
+ @Override
+ @Deprecated
+ public ProvenanceReporter getProvenanceReporter() {
+ return sharedState.getProvenanceReporter();
+ }
+
+ @Override
+ public QueueSize getQueueSize() {
+ return flowFileQueue.size();
+ }
+
+ @Override
+ public Long getCounterValue(final String name) {
+ return sharedState.getCounterValue(name);
+ }
+
+ @Override
+ public int getRemovedCount() {
+ int count = 0;
+ for (final MockProcessSession session : sessionFactory.getCreatedSessions()) {
+ count += session.getRemovedCount();
+ }
+
+ return count;
+ }
+
+ @Override
+ public void setAnnotationData(final String annotationData) {
+ context.setAnnotationData(annotationData);
+ }
+
+ @Override
+ public ValidationResult setProperty(final String propertyName, final String propertyValue) {
+ return context.setProperty(propertyName, propertyValue);
+ }
+
+ @Override
+ public ValidationResult setProperty(final PropertyDescriptor descriptor, final String value) {
+ return context.setProperty(descriptor, value);
+ }
+
+ @Override
+ public ValidationResult setProperty(final PropertyDescriptor descriptor, final AllowableValue value) {
+ return context.setProperty(descriptor, value.getValue());
+ }
+
+ @Override
+ public void setThreadCount(final int threadCount) {
+ if (threadCount > 1 && triggerSerially) {
+ Assert.fail("Cannot set thread-count higher than 1 because the processor is triggered serially");
+ }
+
+ this.numThreads = threadCount;
+ }
+
+ @Override
+ public int getThreadCount() {
+ return numThreads;
+ }
+
+ @Override
+ public void setRelationshipAvailable(final Relationship relationship) {
+ final Set<Relationship> unavailable = new HashSet<>(context.getUnavailableRelationships());
+ unavailable.remove(relationship);
+ context.setUnavailableRelationships(unavailable);
+ }
+
+ @Override
+ public void setRelationshipAvailable(final String relationshipName) {
+ setRelationshipAvailable(new Relationship.Builder().name(relationshipName).build());
+ }
+
+ @Override
+ public void setRelationshipUnavailable(final Relationship relationship) {
+ final Set<Relationship> unavailable = new HashSet<>(context.getUnavailableRelationships());
+ unavailable.add(relationship);
+ context.setUnavailableRelationships(unavailable);
+ }
+
+ @Override
+ public void setRelationshipUnavailable(final String relationshipName) {
+ setRelationshipUnavailable(new Relationship.Builder().name(relationshipName).build());
+ }
+
+ @Override
+ public void addControllerService(final String identifier, final ControllerService service) throws InitializationException {
+ addControllerService(identifier, service, new HashMap<String, String>());
+ }
+
+ @Override
+ public void addControllerService(final String identifier, final ControllerService service, final Map<String, String> properties) throws InitializationException {
+ // hold off on failing due to deprecated annotation for now... will introduce later.
+// for ( final Method method : service.getClass().getMethods() ) {
+// if ( method.isAnnotationPresent(org.apache.nifi.controller.annotation.OnConfigured.class) ) {
+// Assert.fail("Controller Service " + service + " is using deprecated Annotation " + org.apache.nifi.controller.annotation.OnConfigured.class + " for method " + method);
+// }
+// }
+
+ final ComponentLog logger = new MockProcessorLog(identifier, service);
+ final MockControllerServiceInitializationContext initContext = new MockControllerServiceInitializationContext(requireNonNull(service), requireNonNull(identifier), logger);
+ service.initialize(initContext);
+
+ final Map<PropertyDescriptor, String> resolvedProps = new HashMap<>();
+ for (final Map.Entry<String, String> entry : properties.entrySet()) {
+ resolvedProps.put(service.getPropertyDescriptor(entry.getKey()), entry.getValue());
+ }
+
+ try {
+ ReflectionUtils.invokeMethodsWithAnnotation(OnAdded.class, service);
+ } catch (final InvocationTargetException | IllegalAccessException | IllegalArgumentException e) {
+ throw new InitializationException(e);
+ }
+
+ context.addControllerService(identifier, service, resolvedProps, null);
+ }
+
+ @Override
+ public void assertNotValid(final ControllerService service) {
+ final ValidationContext validationContext = new MockValidationContext(context).getControllerServiceValidationContext(service);
+ final Collection<ValidationResult> results = context.getControllerService(service.getIdentifier()).validate(validationContext);
+
+ for (final ValidationResult result : results) {
+ if (!result.isValid()) {
+ return;
+ }
+ }
+
+ Assert.fail("Expected Controller Service " + service + " to be invalid but it is valid");
+ }
+
+ @Override
+ public void assertValid(final ControllerService service) {
+ final ValidationContext validationContext = new MockValidationContext(context).getControllerServiceValidationContext(service);
+ final Collection<ValidationResult> results = context.getControllerService(service.getIdentifier()).validate(validationContext);
+
+ for (final ValidationResult result : results) {
+ if (!result.isValid()) {
+ Assert.fail("Expected Controller Service to be valid but it is invalid due to: " + result.toString());
+ }
+ }
+ }
+
+ @Override
+ public void disableControllerService(final ControllerService service) {
+ final ControllerServiceConfiguration configuration = context.getConfiguration(service.getIdentifier());
+ if (configuration == null) {
+ throw new IllegalArgumentException("Controller Service " + service + " is not known");
+ }
+
+ if (!configuration.isEnabled()) {
+ throw new IllegalStateException("Controller service " + service + " cannot be disabled because it is not enabled");
+ }
+
+ try {
+ ReflectionUtils.invokeMethodsWithAnnotation(OnDisabled.class, service);
+ } catch (final Exception e) {
+ e.printStackTrace();
+ Assert.fail("Failed to disable Controller Service " + service + " due to " + e);
+ }
+
+ configuration.setEnabled(false);
+ }
+
+ @Override
+ public void enableControllerService(final ControllerService service) {
+ final ControllerServiceConfiguration configuration = context.getConfiguration(service.getIdentifier());
+ if (configuration == null) {
+ throw new IllegalArgumentException("Controller Service " + service + " is not known");
+ }
+
+ if (configuration.isEnabled()) {
+ throw new IllegalStateException("Cannot enable Controller Service " + service + " because it is not disabled");
+ }
+
+ try {
+ final ConfigurationContext configContext = new MockConfigurationContext(service, configuration.getProperties(), context);
+ ReflectionUtils.invokeMethodsWithAnnotation(OnEnabled.class, service, configContext);
+ } catch (final InvocationTargetException ite) {
+ ite.getCause().printStackTrace();
+ Assert.fail("Failed to enable Controller Service " + service + " due to " + ite.getCause());
+ } catch (final Exception e) {
+ e.printStackTrace();
+ Assert.fail("Failed to enable Controller Service " + service + " due to " + e);
+ }
+
+ configuration.setEnabled(true);
+ }
+
+ @Override
+ public boolean isControllerServiceEnabled(final ControllerService service) {
+ final ControllerServiceConfiguration configuration = context.getConfiguration(service.getIdentifier());
+ if (configuration == null) {
+ throw new IllegalArgumentException("Controller Service " + service + " is not known");
+ }
+
+ return configuration.isEnabled();
+ }
+
+ @Override
+ public void removeControllerService(final ControllerService service) {
+ disableControllerService(service);
+
+ try {
+ ReflectionUtils.invokeMethodsWithAnnotation(OnRemoved.class, service);
+ } catch (final Exception e) {
+ e.printStackTrace();
+ Assert.fail("Failed to remove Controller Service " + service + " due to " + e);
+ }
+
+ context.removeControllerService(service);
+ }
+
+ @Override
+ public void setAnnotationData(final ControllerService service, final String annotationData) {
+ final ControllerServiceConfiguration configuration = getConfigToUpdate(service);
+ configuration.setAnnotationData(annotationData);
+ }
+
+ private ControllerServiceConfiguration getConfigToUpdate(final ControllerService service) {
+ final ControllerServiceConfiguration configuration = context.getConfiguration(service.getIdentifier());
+ if (configuration == null) {
+ throw new IllegalArgumentException("Controller Service " + service + " is not known");
+ }
+
+ if (configuration.isEnabled()) {
+ throw new IllegalStateException("Controller service " + service + " cannot be modified because it is not disabled");
+ }
+
+ return configuration;
+ }
+
+ @Override
+ public ValidationResult setProperty(final ControllerService service, final PropertyDescriptor property, final AllowableValue value) {
+ return setProperty(service, property, value.getValue());
+ }
+
+ @Override
+ public ValidationResult setProperty(final ControllerService service, final PropertyDescriptor property, final String value) {
+ final ControllerServiceConfiguration configuration = getConfigToUpdate(service);
+ final Map<PropertyDescriptor, String> curProps = configuration.getProperties();
+ final Map<PropertyDescriptor, String> updatedProps = new HashMap<>(curProps);
+
+ final ValidationContext validationContext = new MockValidationContext(context).getControllerServiceValidationContext(service);
+ final ValidationResult validationResult = property.validate(value, validationContext);
+
+ updatedProps.put(property, value);
+ configuration.setProperties(updatedProps);
+
+ return validationResult;
+ }
+
+ @Override
+ public ValidationResult setProperty(final ControllerService service, final String propertyName, final String value) {
+ final PropertyDescriptor descriptor = service.getPropertyDescriptor(propertyName);
+ if (descriptor == null) {
+ return new ValidationResult.Builder()
+ .input(propertyName)
+ .explanation(propertyName + " is not a known Property for Controller Service " + service)
+ .subject("Invalid property")
+ .valid(false)
+ .build();
+ }
+ return setProperty(service, descriptor, value);
+ }
+
+ @Override
+ public ControllerService getControllerService(final String identifier) {
+ return context.getControllerService(identifier);
+ }
+
+ @Override
+ public <T extends ControllerService> T getControllerService(final String identifier, final Class<T> serviceType) {
+ final ControllerService service = context.getControllerService(identifier);
+ return serviceType.cast(service);
+ }
+
+ @Override
+ public boolean removeProperty(PropertyDescriptor descriptor) {
+ return context.removeProperty(descriptor);
+ }
+
+ @Override
+ public List<ProvenanceEventRecord> getProvenanceEvents() {
+ return sharedState.getProvenanceEvents();
+ }
+
+ @Override
+ public void clearProvenanceEvents() {
+ sharedState.clearProvenanceEvents();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-mock/src/main/java/org/apache/nifi/util/TestRunner.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/TestRunner.java b/nifi-mock/src/main/java/org/apache/nifi/util/TestRunner.java
new file mode 100644
index 0000000..fb9fc78
--- /dev/null
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/TestRunner.java
@@ -0,0 +1,765 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.nifi.components.AllowableValue;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.controller.ControllerService;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSessionFactory;
+import org.apache.nifi.processor.Processor;
+import org.apache.nifi.processor.QueueSize;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.provenance.ProvenanceEventRecord;
+import org.apache.nifi.provenance.ProvenanceReporter;
+import org.apache.nifi.reporting.InitializationException;
+
+public interface TestRunner {
+
+ /**
+ * @return the {@link Processor} for which this <code>TestRunner</code> is
+ * configured
+ */
+ Processor getProcessor();
+
+ /**
+ * @return the {@link ProcessSessionFactory} that this
+ * <code>TestRunner</code> will use to invoke the
+ * {@link Processor#onTrigger(ProcessContext, ProcessSessionFactory)} method
+ */
+ ProcessSessionFactory getProcessSessionFactory();
+
+ /**
+ * @return the {@Link ProcessContext} that this <code>TestRunner</code> will
+ * use to invoke the
+ * {@link Processor#onTrigger(ProcessContext, ProcessSessionFactory) onTrigger}
+ * method
+ */
+ ProcessContext getProcessContext();
+
+ /**
+ * Performs exactly the same operation as calling {@link #run(int)} with a
+ * value of 1.
+ */
+ void run();
+
+ /**
+ * Performs the same operation as calling {@link #run(int, boolean)} with a
+ * value of <code>true</code>
+ *
+ * @param iterations number of iterations
+ */
+ void run(int iterations);
+
+ /**
+ * performs the same operation as calling {@link #run(int, boolean, int)}
+ * with a value of {@code iterations}, {@code stopOnFinish}, {@code true}
+ *
+ * @param iterations number of iterations
+ * @param stopOnFinish flag to stop when finished
+ */
+ void run(int iterations, boolean stopOnFinish);
+
+ /**
+ * This method runs the {@link Processor} <code>iterations</code> times,
+ * using the sequence of steps below:
+ * <ul>
+ * <li>
+ * If {@code initialize} is true, run all methods on the Processor that are
+ * annotated with the
+ * {@link nifi.processor.annotation.OnScheduled @OnScheduled} annotation. If
+ * any of these methods throws an Exception, the Unit Test will fail.
+ * </li>
+ * <li>
+ * Schedule the
+ * {@link Processor#onTrigger(ProcessContext, ProcessSessionFactory) onTrigger}
+ * method to be invoked <code>iterations</code> times. The number of threads
+ * used to run these iterations is determined by the ThreadCount of this
+ * <code>TestRunner</code>. By default, the value is set to 1, but it can be
+ * modified by calling the {@link #setThreadCount(int)} method.
+ * </li>
+ * <li>
+ * As soon as the first thread finishes its execution of
+ * {@link Processor#onTrigger(ProcessContext, ProcessSessionFactory) onTrigger},
+ * all methods on the Processor that are annotated with the
+ * {@link nifi.processor.annotation.OnUnscheduled @OnUnscheduled} annotation
+ * are invoked. If any of these methods throws an Exception, the Unit Test
+ * will fail.
+ * </li>
+ * <li>
+ * Waits for all threads to finish execution.
+ * </li>
+ * <li>
+ * If and only if the value of <code>shutdown</code> is true: Call all
+ * methods on the Processor that is annotated with the
+ * {@link nifi.processor.annotation.OnStopped @OnStopped} annotation.
+ * </li>
+ * </ul>
+ *
+ * @param iterations number of iterations
+ * @param stopOnFinish whether or not to run the Processor methods that are
+ * annotated with {@link nifi.processor.annotation.OnStopped @OnStopped}
+ * @param initialize true if must initialize
+ */
+ void run(int iterations, boolean stopOnFinish, final boolean initialize);
+
+ /**
+ * This method runs the {@link Processor} <code>iterations</code> times,
+ * using the sequence of steps below:
+ * <ul>
+ * <li>
+ * If {@code initialize} is true, run all methods on the Processor that are
+ * annotated with the
+ * {@link nifi.processor.annotation.OnScheduled @OnScheduled} annotation. If
+ * any of these methods throws an Exception, the Unit Test will fail.
+ * </li>
+ * <li>
+ * Schedule the
+ * {@link Processor#onTrigger(ProcessContext, ProcessSessionFactory) onTrigger}
+ * method to be invoked <code>iterations</code> times. The number of threads
+ * used to run these iterations is determined by the ThreadCount of this
+ * <code>TestRunner</code>. By default, the value is set to 1, but it can be
+ * modified by calling the {@link #setThreadCount(int)} method.
+ * </li>
+ * <li>
+ * As soon as the first thread finishes its execution of
+ * {@link Processor#onTrigger(ProcessContext, ProcessSessionFactory) onTrigger},
+ * all methods on the Processor that are annotated with the
+ * {@link nifi.processor.annotation.OnUnscheduled @OnUnscheduled} annotation
+ * are invoked. If any of these methods throws an Exception, the Unit Test
+ * will fail.
+ * </li>
+ * <li>
+ * Waits for all threads to finish execution.
+ * </li>
+ * <li>
+ * If and only if the value of <code>shutdown</code> is true: Call all
+ * methods on the Processor that is annotated with the
+ * {@link nifi.processor.annotation.OnStopped @OnStopped} annotation.
+ * </li>
+ * </ul>
+ *
+ * @param iterations number of iterations
+ * @param stopOnFinish whether or not to run the Processor methods that are
+ * annotated with {@link nifi.processor.annotation.OnStopped @OnStopped}
+ * @param initialize true if must initialize
+ * @param runWait indicates the amount of time in milliseconds that the framework should wait for
+ * processors to stop running before calling the {@link nifi.processor.annotation.OnUnscheduled @OnUnscheduled} annotation
+ */
+ void run(int iterations, boolean stopOnFinish, final boolean initialize, final long runWait);
+
+ /**
+ * Invokes all methods on the Processor that are annotated with the
+ * {@link nifi.processor.annotation.OnShutdown @OnShutdown} annotation. If
+ * any of these methods throws an Exception, the Unit Test will fail
+ */
+ void shutdown();
+
+ /**
+ * Updates the number of threads that will be used to run the Processor when
+ * calling the {@link #run()} or {@link #run(int)} methods.
+ *
+ * @param threadCount num threads
+ */
+ void setThreadCount(int threadCount);
+
+ /**
+ * @return the currently configured number of threads that will be used to
+ * runt he Processor when calling the {@link #run()} or {@link #run(int)}
+ * methods
+ */
+ int getThreadCount();
+
+ /**
+ * Updates the value of the property with the given PropertyDescriptor to
+ * the specified value IF and ONLY IF the value is valid according to the
+ * descriptor's validator. Otherwise, Assert.fail() is called, causing the
+ * unit test to fail
+ *
+ * @param propertyName name
+ * @param propertyValue value
+ * @return result
+ */
+ ValidationResult setProperty(String propertyName, String propertyValue);
+
+ /**
+ * Updates the value of the property with the given PropertyDescriptor to
+ * the specified value IF and ONLY IF the value is valid according to the
+ * descriptor's validator. Otherwise, Assert.fail() is called, causing the
+ * unit test to fail
+ *
+ * @param descriptor descriptor
+ * @param value value
+ * @return result
+ */
+ ValidationResult setProperty(PropertyDescriptor descriptor, String value);
+
+ /**
+ * Updates the value of the property with the given PropertyDescriptor to
+ * the specified value IF and ONLY IF the value is valid according to the
+ * descriptor's validator. Otherwise, Assert.fail() is called, causing the
+ * unit test to fail
+ *
+ * @param descriptor descriptor
+ * @param value allowable valu
+ * @return result
+ */
+ ValidationResult setProperty(PropertyDescriptor descriptor, AllowableValue value);
+
+ /**
+ * Sets the annotation data.
+ *
+ * @param annotationData data
+ */
+ void setAnnotationData(String annotationData);
+
+ /**
+ * Asserts that all FlowFiles that were transferred were transferred to the
+ * given relationship
+ *
+ * @param relationship to verify
+ */
+ void assertAllFlowFilesTransferred(String relationship);
+
+ /**
+ * Asserts that all FlowFiles that were transferred were transferred to the
+ * given relationship
+ *
+ * @param relationship to verify
+ */
+ void assertAllFlowFilesTransferred(Relationship relationship);
+
+ /**
+ * Asserts that all FlowFiles that were transferred were transferred to the
+ * given relationship and that the number of FlowFiles transferred is equal
+ * to <code>count</code>
+ *
+ * @param relationship to verify
+ * @param count number of expected transfers
+ */
+ void assertAllFlowFilesTransferred(String relationship, int count);
+
+ /**
+ * Asserts that all FlowFiles that were transferred were transferred to the
+ * given relationship and that the number of FlowFiles transferred is equal
+ * to <code>count</code>
+ *
+ * @param relationship to verify
+ * @param count number of expected transfers
+ */
+ void assertAllFlowFilesTransferred(Relationship relationship, int count);
+
+ /**
+ * Assert that the number of FlowFiles transferred to the given relationship
+ * is equal to the given count
+ *
+ * @param relationship to verify
+ * @param count number of expected transfers
+ */
+ void assertTransferCount(Relationship relationship, int count);
+
+ /**
+ * Assert that the number of FlowFiles transferred to the given relationship
+ * is equal to the given count
+ *
+ * @param relationship to verify
+ * @param count number of expected transfers
+ */
+ void assertTransferCount(String relationship, int count);
+
+ /**
+ * Assert that there are no FlowFiles left on the input queue.
+ */
+ void assertQueueEmpty();
+
+ /**
+ * @return <code>true</code> if the Input Queue to the Processor is empty,
+ * <code>false</code> otherwise
+ */
+ boolean isQueueEmpty();
+
+ /**
+ * Assert that there is at least one FlowFile left on the input queue.
+ */
+ void assertQueueNotEmpty();
+
+ /**
+ * Assert that the currently configured set of properties/annotation data
+ * are valid
+ */
+ void assertValid();
+
+ /**
+ * Assert that the currently configured set of properties/annotation data
+ * are NOT valid
+ */
+ void assertNotValid();
+
+ /**
+ * Resets the Transfer Counts that indicate how many FlowFiles have been
+ * transferred to each Relationship and removes from memory any FlowFiles
+ * that have been transferred to this Relationships. This method should be
+ * called between successive calls to {@link #run(int) run} if the output is
+ * to be examined after each run.
+ */
+ void clearTransferState();
+
+ /**
+ * Enqueues the given FlowFiles into the Processor's input queue
+ *
+ * @param flowFiles to enqueue
+ */
+ void enqueue(FlowFile... flowFiles);
+
+ /**
+ * Reads the content from the given {@link Path} into memory and creates a
+ * FlowFile from this content with no attributes and adds this FlowFile to
+ * the Processor's Input Queue
+ *
+ * @param path to read content from
+ * @throws IOException if unable to read content
+ */
+ void enqueue(Path path) throws IOException;
+
+ /**
+ * Reads the content from the given {@link Path} into memory and creates a
+ * FlowFile from this content with the given attributes and adds this
+ * FlowFile to the Processor's Input Queue
+ *
+ * @param path to read content from
+ * @param attributes attributes to use for new flow file
+ * @throws IOException if unable to read content
+ */
+ void enqueue(Path path, Map<String, String> attributes) throws IOException;
+
+ /**
+ * Copies the content from the given byte array into memory and creates a
+ * FlowFile from this content with no attributes and adds this FlowFile to
+ * the Processor's Input Queue
+ *
+ * @param data to enqueue
+ */
+ void enqueue(byte[] data);
+
+ /**
+ * Copies the content from the given byte array into memory and creates a
+ * FlowFile from this content with the given attributes and adds this
+ * FlowFile to the Processor's Input Queue
+ *
+ * @param data to enqueue
+ * @param attributes to use for enqueued items
+ */
+ void enqueue(byte[] data, Map<String, String> attributes);
+
+ /**
+ * Reads the content from the given {@link InputStream} into memory and
+ * creates a FlowFile from this content with no attributes and adds this
+ * FlowFile to the Processor's Input Queue
+ *
+ * @param data to source data from
+ */
+ void enqueue(InputStream data);
+
+ /**
+ * Reads the content from the given {@link InputStream} into memory and
+ * creates a FlowFile from this content with the given attributes and adds
+ * this FlowFile to the Processor's Input Queue
+ *
+ * @param data source of data
+ * @param attributes to use for flow files
+ */
+ void enqueue(InputStream data, Map<String, String> attributes);
+
+ /**
+ * Copies the contents of the given {@link MockFlowFile} into a byte array
+ * and returns that byte array.
+ *
+ * @param flowFile to get content for
+ * @return byte array of flowfile content
+ */
+ byte[] getContentAsByteArray(MockFlowFile flowFile);
+
+ /**
+ * Returns a List of FlowFiles in the order in which they were transferred
+ * to the given relationship
+ *
+ * @param relationship to get flowfiles for
+ * @return flowfiles transfered to given relationship
+ */
+ List<MockFlowFile> getFlowFilesForRelationship(String relationship);
+
+ /**
+ * Returns a List of FlowFiles in the order in which they were transferred
+ * to the given relationship
+ *
+ * @param relationship to get flowfiles for
+ * @return flowfiles transfered to given relationship
+ */
+ List<MockFlowFile> getFlowFilesForRelationship(Relationship relationship);
+
+ /**
+ * @return the {@link ProvenanceReporter} that will be used by the
+ * configured {@link Processor} for reporting Provenance Events
+ */
+ ProvenanceReporter getProvenanceReporter();
+
+ /**
+ * @return the current size of the Processor's Input Queue
+ */
+ QueueSize getQueueSize();
+
+ /**
+ * @param name of counter
+ * @return the current value of the counter with the specified name, or null
+ * if no counter exists with the specified name
+ */
+ Long getCounterValue(String name);
+
+ /**
+ * @return the number of FlowFiles that have been removed from the system
+ */
+ int getRemovedCount();
+
+ /**
+ * Indicates to the Framework that the given Relationship should be
+ * considered "available", meaning that the queues of all Connections that
+ * contain this Relationship are not full. This is generally used only when
+ * dealing with Processors that use the
+ * {@link nifi.processor.annotation.TriggerWhenAnyDestinationAvailable}
+ * annotation.
+ *
+ * @param relationship to mark as available
+ */
+ void setRelationshipAvailable(Relationship relationship);
+
+ /**
+ * Indicates to the Framework that the given Relationship with the given
+ * name should be considered "available", meaning that the queues of all
+ * Connections that contain this Relationship are not full. This is
+ * generally used only when dealing with Processors that use the
+ * {@link nifi.processor.annotation.TriggerWhenAnyDestinationAvailable}
+ *
+ * @param relationshipName relationship name
+ */
+ void setRelationshipAvailable(String relationshipName);
+
+ /**
+ * Indicates to the Framework that the given Relationship should NOT be
+ * considered "available", meaning that the queue of at least one Connection
+ * that contain this Relationship is full. This is generally used only when
+ * dealing with Processors that use the
+ * {@link nifi.processor.annotation.TriggerWhenAnyDestinationAvailable}
+ * annotation.
+ *
+ * @param relationship to mark as unavailable
+ */
+ void setRelationshipUnavailable(Relationship relationship);
+
+ /**
+ * Indicates to the Framework that the Relationship with the given name
+ * should NOT be considered "available", meaning that the queue of at least
+ * one Connection that contain this Relationship is full. This is generally
+ * used only when dealing with Processors that use the
+ * {@link nifi.processor.annotation.TriggerWhenAnyDestinationAvailable}
+ *
+ * @param relationshipName name of relationship.
+ */
+ void setRelationshipUnavailable(String relationshipName);
+
+ /**
+ * Adds the given {@link ControllerService} to this TestRunner so that the
+ * configured Processor can access it using the given
+ * <code>identifier</code>. The ControllerService is not expected to be
+ * initialized, as the framework will create the appropriate
+ * {@link nifi.controller.ControllerServiceInitializationContext ControllerServiceInitializationContext}
+ * and initialize the ControllerService with no specified properties.
+ *
+ * This will call any method on the given Controller Service that is
+ * annotated with the
+ * {@link org.apache.nifi.annotation.lifecycle.OnAdded @OnAdded} annotation.
+ *
+ * @param identifier of service
+ * @param service the service
+ * @throws InitializationException ie
+ */
+ void addControllerService(String identifier, ControllerService service) throws InitializationException;
+
+ /**
+ * Adds the given {@link ControllerService} to this TestRunner so that the
+ * configured Processor can access it using the given
+ * <code>identifier</code>. The ControllerService is not expected to be
+ * initialized, as the framework will create the appropriate
+ * {@link nifi.controller.ControllerServiceInitializationContext ControllerServiceInitializationContext}
+ * and initialize the ControllerService with the given properties.
+ *
+ * This will call any method on the given Controller Service that is
+ * annotated with the
+ * {@link org.apache.nifi.annotation.lifecycle.OnAdded @OnAdded} annotation.
+ *
+ * @param identifier of service
+ * @param service the service
+ * @param properties service properties
+ * @throws InitializationException ie
+ */
+ void addControllerService(String identifier, ControllerService service, Map<String, String> properties) throws InitializationException;
+
+ /**
+ * <p>
+ * Marks the Controller Service as enabled so that it can be used by other
+ * components.
+ * </p>
+ *
+ * <p>
+ * This method will result in calling any method in the Controller Service
+ * that is annotated with the
+ * {@link org.apache.nifi.annotation.lifecycle.OnEnabled @OnEnabled}
+ * annotation.
+ * </p>
+ *
+ * @param service the service to enable
+ */
+ void enableControllerService(ControllerService service);
+
+ /**
+ * <p>
+ * Marks the Controller Service as disabled so that it cannot be used by
+ * other components.
+ * </p>
+ *
+ * <p>
+ * This method will result in calling any method in the Controller Service
+ * that is annotated with the
+ * {@link org.apache.nifi.annotation.lifecycle.OnDisabled @OnDisabled}
+ * annotation.
+ * </p>
+ *
+ * @param service the service to disable
+ */
+ void disableControllerService(ControllerService service);
+
+ /**
+ * @param service the service
+ * @return {@code true} if the given Controller Service is enabled,
+ * {@code false} if it is disabled
+ *
+ * @throws IllegalArgumentException if the given ControllerService is not
+ * known by this TestRunner (i.e., it has not been added via the
+ * {@link #addControllerService(String, ControllerService)} or
+ * {@link #addControllerService(String, ControllerService, Map)} method or
+ * if the Controller Service has been removed via the
+ * {@link #removeControllerService(ControllerService)} method.
+ */
+ boolean isControllerServiceEnabled(ControllerService service);
+
+ /**
+ * <p>
+ * Removes the Controller Service from the TestRunner. This will call any
+ * method on the ControllerService that is annotated with the
+ * {@link org.apache.nifi.annotation.lifecycle.OnRemoved @OnRemoved}
+ * annotation.
+ * </p>
+ *
+ * @param service the service
+ *
+ * @throws IllegalStateException if the ControllerService is not disabled
+ * @throws IllegalArgumentException if the given ControllerService is not
+ * known by this TestRunner (i.e., it has not been added via the
+ * {@link #addControllerService(String, ControllerService)} or
+ * {@link #addControllerService(String, ControllerService, Map)} method or
+ * if the Controller Service has been removed via the
+ * {@link #removeControllerService(ControllerService)} method.
+ *
+ */
+ void removeControllerService(ControllerService service);
+
+ /**
+ * Sets the given property on the given ControllerService
+ *
+ * @param service to modify
+ * @param property to modify
+ * @param value value to use
+ * @return result
+ *
+ * @throws IllegalStateException if the ControllerService is not disabled
+ * @throws IllegalArgumentException if the given ControllerService is not
+ * known by this TestRunner (i.e., it has not been added via the
+ * {@link #addControllerService(String, ControllerService)} or
+ * {@link #addControllerService(String, ControllerService, Map)} method or
+ * if the Controller Service has been removed via the
+ * {@link #removeControllerService(ControllerService)} method.
+ *
+ */
+ ValidationResult setProperty(ControllerService service, PropertyDescriptor property, String value);
+
+ /**
+ * Sets the given property on the given ControllerService
+ *
+ * @param service to modify
+ * @param property to modify
+ * @param value value to use
+ * @return result
+ *
+ * @throws IllegalStateException if the ControllerService is not disabled
+ * @throws IllegalArgumentException if the given ControllerService is not
+ * known by this TestRunner (i.e., it has not been added via the
+ * {@link #addControllerService(String, ControllerService)} or
+ * {@link #addControllerService(String, ControllerService, Map)} method or
+ * if the Controller Service has been removed via the
+ * {@link #removeControllerService(ControllerService)} method.
+ *
+ */
+ ValidationResult setProperty(ControllerService service, PropertyDescriptor property, AllowableValue value);
+
+ /**
+ * Sets the property with the given name on the given ControllerService
+ *
+ * @param service to modify
+ * @param propertyName to modify
+ * @param value value to use
+ * @return result
+ *
+ * @throws IllegalStateException if the ControllerService is not disabled
+ * @throws IllegalArgumentException if the given ControllerService is not
+ * known by this TestRunner (i.e., it has not been added via the
+ * {@link #addControllerService(String, ControllerService)} or
+ * {@link #addControllerService(String, ControllerService, Map)} method or
+ * if the Controller Service has been removed via the
+ * {@link #removeControllerService(ControllerService)} method.
+ *
+ */
+ ValidationResult setProperty(ControllerService service, String propertyName, String value);
+
+ /**
+ * Sets the annontation data of the given service to the provided annotation
+ * data.
+ *
+ * @param service to modify
+ * @param annotationData the data
+ *
+ * @throws IllegalStateException if the Controller Service is not disabled
+ *
+ * @throws IllegalArgumentException if the given ControllerService is not
+ * known by this TestRunner (i.e., it has not been added via the
+ * {@link #addControllerService(String, ControllerService)} or
+ * {@link #addControllerService(String, ControllerService, Map)} method or
+ * if the Controller Service has been removed via the
+ * {@link #removeControllerService(ControllerService)} method.
+ */
+ void setAnnotationData(ControllerService service, String annotationData);
+
+ /**
+ * @param identifier of controller service
+ * @return the {@link ControllerService} that is registered with the given
+ * identifier, or <code>null</code> if no Controller Service exists with the
+ * given identifier
+ */
+ ControllerService getControllerService(String identifier);
+
+ /**
+ * Assert that the currently configured set of properties/annotation data
+ * are valid for the given Controller Service.
+ *
+ * @param service the service to validate
+ * @throws IllegalArgumentException if the given ControllerService is not
+ * known by this TestRunner (i.e., it has not been added via the
+ * {@link #addControllerService(String, ControllerService)} or
+ * {@link #addControllerService(String, ControllerService, Map)} method or
+ * if the Controller Service has been removed via the
+ * {@link #removeControllerService(ControllerService)} method.
+ */
+ void assertValid(ControllerService service);
+
+ /**
+ * Assert that the currently configured set of properties/annotation data
+ * are NOT valid for the given Controller Service.
+ *
+ * @param service the service to validate
+ * @throws IllegalArgumentException if the given ControllerService is not
+ * known by this TestRunner (i.e., it has not been added via the
+ * {@link #addControllerService(String, ControllerService)} or
+ * {@link #addControllerService(String, ControllerService, Map)} method or
+ * if the Controller Service has been removed via the
+ * {@link #removeControllerService(ControllerService)} method.
+ *
+ */
+ void assertNotValid(ControllerService service);
+
+ /**
+ * @param <T> type of service
+ * @param identifier identifier of service
+ * @param serviceType type of service
+ * @return the {@link ControllerService} that is registered with the given
+ * identifier, cast as the provided service type, or <code>null</code> if no
+ * Controller Service exists with the given identifier
+ *
+ * @throws ClassCastException if the identifier given is registered for a
+ * Controller Service but that Controller Service is not of the type
+ * specified
+ */
+ <T extends ControllerService> T getControllerService(String identifier, Class<T> serviceType);
+
+ /**
+ * Specifies whether or not the TestRunner will validate the use of
+ * Expression Language. By default, the value is <code>true</code>, which
+ * means that an Exception will be thrown if the Processor attempts to
+ * obtain the configured value of a Property without calling
+ * {@link nifi.components.PropertyValue#evaluateAttributeExpressions evaluateAttributeExpressions}
+ * on the Property Value or if
+ * {@link nifi.components.PropertyValue#evaluateAttributeExpressions evaluateAttributeExpressions}
+ * is called but the PropertyDescriptor indicates that the Expression
+ * Language is not supported.
+ *
+ * <p>
+ * <b>See Also:
+ * </b>{@link PropertyDescriptor.Builder#expressionLanguageSupported(boolean)}
+ * </p>
+ *
+ * @param validate whether there is any need to validate the EL was used
+ */
+ void setValidateExpressionUsage(boolean validate);
+
+ /**
+ * Removes the {@link PropertyDescriptor} from the {@link ProcessContext},
+ * effectively setting its value to null.
+ *
+ * @param descriptor of property to remove
+ * @return true if removed
+ */
+ boolean removeProperty(PropertyDescriptor descriptor);
+
+ /**
+ * Returns a {@link List} of all {@link ProvenanceEventRecord}s that were
+ * emitted by the Processor
+ *
+ * @return a List of all Provenance Events that were emitted by the
+ * Processor
+ */
+ List<ProvenanceEventRecord> getProvenanceEvents();
+
+ /**
+ * Clears the Provenance Events that have been emitted by the Processor
+ */
+ void clearProvenanceEvents();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-mock/src/main/java/org/apache/nifi/util/TestRunners.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/TestRunners.java b/nifi-mock/src/main/java/org/apache/nifi/util/TestRunners.java
new file mode 100644
index 0000000..f2b0b23
--- /dev/null
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/TestRunners.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import org.apache.nifi.processor.Processor;
+
+public class TestRunners {
+
+ public static TestRunner newTestRunner(final Processor processor) {
+ return new StandardProcessorTestRunner(processor);
+ }
+
+ public static TestRunner newTestRunner(final Class<? extends Processor> processorClass) {
+ try {
+ return newTestRunner(processorClass.newInstance());
+
+ } catch (final Exception e) {
+ System.err.println("Could not instantiate instance of class " + processorClass.getName() + " due to: " + e);
+ throw new RuntimeException(e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-mock/src/test/java/org/apache/nifi/util/CurrentTestStandardProcessorTestRunner.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/test/java/org/apache/nifi/util/CurrentTestStandardProcessorTestRunner.java b/nifi-mock/src/test/java/org/apache/nifi/util/CurrentTestStandardProcessorTestRunner.java
new file mode 100644
index 0000000..6b403af
--- /dev/null
+++ b/nifi-mock/src/test/java/org/apache/nifi/util/CurrentTestStandardProcessorTestRunner.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class CurrentTestStandardProcessorTestRunner {
+
+ /**
+ * This test will verify that all iterations of the run are finished before unscheduled is called
+ */
+ @Test
+ public void testOnScheduledCalledAfterRunFinished() {
+ SlowRunProcessor processor = new SlowRunProcessor();
+ StandardProcessorTestRunner runner = new StandardProcessorTestRunner(processor);
+ final int iterations = 5;
+ runner.run(iterations);
+ // if the counter is not equal to iterations, the the processor must have been unscheduled
+ // before all the run calls were made, that would be bad.
+ Assert.assertEquals(iterations, processor.getCounter());
+ }
+
+ /**
+ * This processor simulates a "slow" processor that checks whether it is scheduled before doing something
+ *
+ *
+ */
+ private static class SlowRunProcessor extends AbstractProcessor {
+
+ private int counter = 0;
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+
+ try {
+ // be slow
+ Thread.sleep(50);
+ // make sure we are still scheduled
+ if (isScheduled()) {
+ // increment counter
+ ++counter;
+ }
+ } catch (InterruptedException e) {
+ }
+
+ }
+
+ public int getCounter() {
+ return counter;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-mock/src/test/java/org/apache/nifi/util/TestStandardProcessorTestRunner.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/test/java/org/apache/nifi/util/TestStandardProcessorTestRunner.java b/nifi-mock/src/test/java/org/apache/nifi/util/TestStandardProcessorTestRunner.java
new file mode 100644
index 0000000..323a357
--- /dev/null
+++ b/nifi-mock/src/test/java/org/apache/nifi/util/TestStandardProcessorTestRunner.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util;
+
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.junit.Ignore;
+import org.junit.Test;
+
+@Ignore("This should not be enabled until we actually fail processor unit tests for using deprecated methods, which should happen in 0.1.0")
+public class TestStandardProcessorTestRunner {
+
+ @Test(expected = AssertionError.class)
+ public void testFailOnDeprecatedTypeAnnotation() {
+ new StandardProcessorTestRunner(new DeprecatedAnnotation());
+ }
+
+ @Test
+ public void testDoesNotFailOnNonDeprecatedTypeAnnotation() {
+ new StandardProcessorTestRunner(new NewAnnotation());
+ }
+
+ @Test(expected = AssertionError.class)
+ public void testFailOnDeprecatedMethodAnnotation() {
+ new StandardProcessorTestRunner(new DeprecatedMethodAnnotation());
+ }
+
+ @Test
+ public void testDoesNotFailOnNonDeprecatedMethodAnnotation() {
+ new StandardProcessorTestRunner(new NewMethodAnnotation());
+ }
+
+ @SuppressWarnings("deprecation")
+ @org.apache.nifi.processor.annotation.Tags({"deprecated"})
+ private static class DeprecatedAnnotation extends AbstractProcessor {
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ }
+ }
+
+ @org.apache.nifi.annotation.documentation.Tags({"deprecated"})
+ private static class NewAnnotation extends AbstractProcessor {
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ }
+ }
+
+ private static class NewMethodAnnotation extends AbstractProcessor {
+
+ @org.apache.nifi.annotation.lifecycle.OnScheduled
+ public void dummy() {
+
+ }
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ }
+ }
+
+ private static class DeprecatedMethodAnnotation extends AbstractProcessor {
+
+ @SuppressWarnings("deprecation")
+ @org.apache.nifi.processor.annotation.OnScheduled
+ public void dummy() {
+
+ }
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ }
+ }
+}
[43/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/reporting/ReportingContext.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/reporting/ReportingContext.java b/nifi-api/src/main/java/org/apache/nifi/reporting/ReportingContext.java
new file mode 100644
index 0000000..281194c
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/reporting/ReportingContext.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.reporting;
+
+import java.util.Map;
+
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.PropertyValue;
+import org.apache.nifi.controller.ControllerServiceLookup;
+
+/**
+ * This interface provides a bridge between the NiFi Framework and a
+ * {@link ReportingTask}. This context allows a ReportingTask to access
+ * statistics, metrics, and monitoring information, as well as configuration
+ * supplied by the user.
+ */
+public interface ReportingContext {
+
+ /**
+ * @return a Map of all known {@link PropertyDescriptor}s to their
+ * configured properties. This Map will contain a <code>null</code> for any
+ * Property that has not been configured by the user, even if the
+ * PropertyDescriptor has a default value
+ */
+ Map<PropertyDescriptor, String> getProperties();
+
+ /**
+ * @param propertyName descriptor of property to lookup the value of
+ * @return PropertyValue that represents the user-configured value for the given
+ * {@link PropertyDescriptor}
+ */
+ PropertyValue getProperty(PropertyDescriptor propertyName);
+
+ /**
+ * @return the {@link EventAccess} object that can be used to obtain
+ * information about specific events and reports that have happened
+ */
+ EventAccess getEventAccess();
+
+ /**
+ * @return the {@link BulletinRepository} that can be used to analyze
+ * Bulletins that have been emitted and register new Bulletins
+ */
+ BulletinRepository getBulletinRepository();
+
+ /**
+ * Creates a system-level {@link Bulletin} with the given category, severity
+ * level, and message, so that the Bulletin can be added to the
+ * {@link BulletinRepository}.
+ *
+ * @param category of bulletin
+ * @param severity of bulletin
+ * @param message of bulletin
+ * @return new bulletin
+ */
+ Bulletin createBulletin(String category, Severity severity, String message);
+
+ /**
+ * Creates a {@link Bulletin} for the component with the specified
+ * identifier
+ *
+ * @param componentId the ID of the component
+ * @param category the name of the bulletin's category
+ * @param severity the severity level of the bulletin
+ * @param message the bulletin's message
+ * @return new bulletin
+ */
+ Bulletin createBulletin(String componentId, String category, Severity severity, String message);
+
+ /**
+ * @return the {@link ControllerServiceLookup} which can be used to obtain
+ * Controller Services
+ */
+ ControllerServiceLookup getControllerServiceLookup();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/reporting/ReportingInitializationContext.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/reporting/ReportingInitializationContext.java b/nifi-api/src/main/java/org/apache/nifi/reporting/ReportingInitializationContext.java
new file mode 100644
index 0000000..d014b26
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/reporting/ReportingInitializationContext.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.reporting;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.nifi.controller.ControllerServiceLookup;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.scheduling.SchedulingStrategy;
+
+/**
+ * A ReportingConfiguration provides configuration information to a
+ * ReportingTask at the time of initialization
+ */
+public interface ReportingInitializationContext {
+
+ /**
+ * @return the identifier for this ReportingTask
+ */
+ String getIdentifier();
+
+ /**
+ * @return the configured name for this ReportingTask
+ */
+ String getName();
+
+ /**
+ * Returns the amount of time, in the given {@link TimeUnit} that will
+ * elapsed between the return of one execution of the
+ * {@link ReportingTask}'s
+ * {@link ReportingTask#onTrigger(ReportingContext) onTrigger} method and
+ * the time at which the method is invoked again. This method will return
+ * <code>-1L</code> if the Scheduling Strategy is not set to
+ * {@link SchedulingStrategy#TIMER_DRIVEN}
+ *
+ * @param timeUnit unit of time for scheduling
+ * @return period of time
+ */
+ long getSchedulingPeriod(TimeUnit timeUnit);
+
+ /**
+ * @return the {@link ControllerServiceLookup} which can be used to obtain
+ * Controller Services
+ */
+ ControllerServiceLookup getControllerServiceLookup();
+
+ /**
+ * @return a String representation of the scheduling period
+ */
+ String getSchedulingPeriod();
+
+ /**
+ * @return the {@link SchedulingStrategy} that is used to trigger the task
+ * to run
+ */
+ SchedulingStrategy getSchedulingStrategy();
+
+ /**
+ * @return a logger that can be used to log important events in a standard
+ * way and generate bulletins when appropriate
+ */
+ ComponentLog getLogger();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/reporting/ReportingTask.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/reporting/ReportingTask.java b/nifi-api/src/main/java/org/apache/nifi/reporting/ReportingTask.java
new file mode 100644
index 0000000..3de9b93
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/reporting/ReportingTask.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.reporting;
+
+import org.apache.nifi.components.ConfigurableComponent;
+import org.apache.nifi.controller.annotation.OnConfigured;
+
+/**
+ * Defines a task that is responsible for reporting status information to
+ * external destinations. All implementations of this class must have a default
+ * constructor.
+ *
+ * <p>
+ * <code>ReportingTask</code>s are discovered using Java's
+ * <code>ServiceLoader</code> mechanism. As a result, all implementations must
+ * follow these rules:
+ *
+ * <ul>
+ * <li>The implementation must implement this interface.</li>
+ * <li>The implementation must have a file named
+ * org.apache.nifi.reporting.ReportingTask located within the jar's
+ * <code>META-INF/services</code> directory. This file contains a list of
+ * fully-qualified class names of all <code>ReportingTask</code>s in the jar,
+ * one-per-line.
+ * <li>The implementation must support a default constructor.</li>
+ * </ul>
+ * </p>
+ *
+ * <p>
+ * ReportingTasks are scheduled on a delayed interval with a single thread.
+ * Therefore, implementations are not required to be thread-safe.
+ * </p>
+ *
+ * <p>
+ * ReportingTasks may choose to annotate a method with the
+ * {@link OnConfigured @OnConfigured} annotation. If this is done, that method
+ * will be invoked after all properties have been set for the ReportingTask and
+ * before it is scheduled to run. If the method throws an Exception, the
+ * ReportingTask will be Administratively yielded and will not run for the
+ * configured period of time.
+ * </p>
+ */
+public interface ReportingTask extends ConfigurableComponent {
+
+ /**
+ * Provides the Reporting Task with access to objects that may be of use
+ * throughout the life of the service
+ *
+ * @param config of initialization context
+ * @throws org.apache.nifi.reporting.InitializationException if unable to init
+ */
+ void initialize(ReportingInitializationContext config) throws InitializationException;
+
+ /**
+ * This method is called on a scheduled interval to allow the Reporting Task
+ * to perform its tasks.
+ *
+ * @param context reporting context
+ */
+ void onTrigger(ReportingContext context);
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/reporting/Severity.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/reporting/Severity.java b/nifi-api/src/main/java/org/apache/nifi/reporting/Severity.java
new file mode 100644
index 0000000..8c2b27f
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/reporting/Severity.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.reporting;
+
+public enum Severity {
+
+ INFO,
+ WARNING,
+ ERROR;
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/scheduling/SchedulingStrategy.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/scheduling/SchedulingStrategy.java b/nifi-api/src/main/java/org/apache/nifi/scheduling/SchedulingStrategy.java
new file mode 100644
index 0000000..ccf4281
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/scheduling/SchedulingStrategy.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.scheduling;
+
+/**
+ * Defines a Scheduling Strategy to use when scheduling Components (Ports,
+ * Funnels, Processors) to run
+ */
+public enum SchedulingStrategy {
+
+ /**
+ * Components should be scheduled to run whenever a relevant Event occurs.
+ * Examples of "relevant Events" are:
+ *
+ * <ul>
+ * <li>A FlowFile is added to one of the Component's incoming
+ * Connections</li>
+ * <li>A FlowFile is removed from one of the Component's outgoing
+ * Connections</li>
+ * <li>The Component is scheduled to run (started)</li>
+ * </ul>
+ *
+ * <p>
+ * When using this mode, the user will be unable to configure the scheduling
+ * period. Instead, the framework will manage this.
+ * </p>
+ *
+ * <p>
+ * When using this mode, the maximum number of concurrent tasks can be set
+ * to 0, indicating no maximum.
+ * </p>
+ *
+ * <p>
+ * Not all Components support Event-Driven mode.
+ * </p>
+ */
+ EVENT_DRIVEN(0, null),
+ /**
+ * Components should be scheduled to run on a periodic interval that is
+ * user-defined with a user-defined number of concurrent tasks. All
+ * Components support Timer-Driven mode.
+ */
+ TIMER_DRIVEN(1, "0 sec"),
+ /**
+ * Indicates that the component will be scheduled via timer only on the
+ * Primary Node. If the instance is not part of a cluster and this
+ * Scheduling Strategy is used, the component will be scheduled in the same
+ * manner as if {@link TIMER_DRIVEN} were used.
+ */
+ PRIMARY_NODE_ONLY(1, "0 sec"),
+ /**
+ * Indicates that the component will be scheduled to run according to a
+ * Cron-style expression
+ */
+ CRON_DRIVEN(1, "* * * * * ?");
+
+ private final int defaultConcurrentTasks;
+ private final String defaultSchedulingPeriod;
+
+ private SchedulingStrategy(final int defaultConcurrentTasks, final String defaultSchedulingPeriod) {
+ this.defaultConcurrentTasks = defaultConcurrentTasks;
+ this.defaultSchedulingPeriod = defaultSchedulingPeriod;
+ }
+
+ public int getDefaultConcurrentTasks() {
+ return defaultConcurrentTasks;
+ }
+
+ public String getDefaultSchedulingPeriod() {
+ return defaultSchedulingPeriod;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/search/SearchContext.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/search/SearchContext.java b/nifi-api/src/main/java/org/apache/nifi/search/SearchContext.java
new file mode 100644
index 0000000..1da8c90
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/search/SearchContext.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.search;
+
+import java.util.Map;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.PropertyValue;
+
+/**
+ *
+ */
+public interface SearchContext {
+
+ /**
+ * @return the search term
+ */
+ String getSearchTerm();
+
+ /**
+ * @return the annotation data
+ */
+ String getAnnotationData();
+
+ /**
+ * @param property to get value of
+ * @return a PropertyValue that encapsulates the value configured for the
+ * given PropertyDescriptor
+ */
+ PropertyValue getProperty(PropertyDescriptor property);
+
+ /**
+ * @return a Map of all configured Properties
+ */
+ Map<PropertyDescriptor, String> getProperties();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/search/SearchResult.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/search/SearchResult.java b/nifi-api/src/main/java/org/apache/nifi/search/SearchResult.java
new file mode 100644
index 0000000..29e490a
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/search/SearchResult.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.search;
+
+/**
+ *
+ */
+public class SearchResult {
+
+ private final String label;
+ private final String match;
+
+ private SearchResult(final Builder builder) {
+ this.label = builder.label;
+ this.match = builder.match;
+ }
+
+ /**
+ * @return the label for this search result
+ */
+ public String getLabel() {
+ return label;
+ }
+
+ /**
+ * @return the matching string for this search result
+ */
+ public String getMatch() {
+ return match;
+ }
+
+ public static final class Builder {
+
+ private String label;
+ private String match;
+
+ /**
+ * Set the label for the search result.
+ *
+ * @param label to set
+ * @return the builder
+ */
+ public Builder label(final String label) {
+ this.label = label;
+ return this;
+ }
+
+ /**
+ * Set the matching string for the search result.
+ *
+ * @param match string
+ * @return the builder
+ */
+ public Builder match(final String match) {
+ this.match = match;
+ return this;
+ }
+
+ public SearchResult build() {
+ return new SearchResult(this);
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/search/Searchable.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/search/Searchable.java b/nifi-api/src/main/java/org/apache/nifi/search/Searchable.java
new file mode 100644
index 0000000..64f32f9
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/search/Searchable.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.search;
+
+import java.util.Collection;
+
+/**
+ *
+ */
+public interface Searchable {
+
+ Collection<SearchResult> search(SearchContext context);
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/web/ClusterRequestException.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/web/ClusterRequestException.java b/nifi-api/src/main/java/org/apache/nifi/web/ClusterRequestException.java
new file mode 100644
index 0000000..ee5f417
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/web/ClusterRequestException.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web;
+
+/**
+ * An general error occurred when attempting to communicate with the cluster.
+ */
+public class ClusterRequestException extends RuntimeException {
+
+ public ClusterRequestException(Throwable cause) {
+ super(cause);
+ }
+
+ public ClusterRequestException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public ClusterRequestException(String message) {
+ super(message);
+ }
+
+ public ClusterRequestException() {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/web/ComponentDetails.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/web/ComponentDetails.java b/nifi-api/src/main/java/org/apache/nifi/web/ComponentDetails.java
new file mode 100644
index 0000000..5614fc2
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/web/ComponentDetails.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web;
+
+import java.util.Collection;
+import java.util.Map;
+
+/**
+ * Details about a given component. Contains configuration and current
+ * validation errors.
+ */
+public class ComponentDetails {
+
+ private final String id;
+ private final String name;
+ private final String type;
+ private final String state;
+ private final String annotationData;
+ private final Map<String, String> properties;
+ private final Collection<String> validationErrors;
+
+ private ComponentDetails(final Builder builder) {
+ this.id = builder.id;
+ this.name = builder.name;
+ this.type = builder.type;
+ this.state = builder.state;
+ this.annotationData = builder.annotationData;
+ this.properties = builder.properties;
+ this.validationErrors = builder.validationErrors;
+ }
+
+ /**
+ * @return component id
+ */
+ public String getId() {
+ return id;
+ }
+
+ /**
+ * @return component name
+ */
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * @return component type
+ */
+ public String getType() {
+ return type;
+ }
+
+ /**
+ * @return component state
+ */
+ public String getState() {
+ return state;
+ }
+
+ /**
+ * @return component's annotation data
+ */
+ public String getAnnotationData() {
+ return annotationData;
+ }
+
+ /**
+ * @return Mapping of component properties
+ */
+ public Map<String, String> getProperties() {
+ return properties;
+ }
+
+ /**
+ * @return Current validation errors for the component
+ */
+ public Collection<String> getValidationErrors() {
+ return validationErrors;
+ }
+
+ public static final class Builder {
+
+ private String id;
+ private String name;
+ private String type;
+ private String state;
+ private String annotationData;
+ private Map<String, String> properties;
+ private Collection<String> validationErrors;
+
+ public Builder id(final String id) {
+ this.id = id;
+ return this;
+ }
+
+ public Builder name(final String name) {
+ this.name = name;
+ return this;
+ }
+
+ public Builder type(final String type) {
+ this.type = type;
+ return this;
+ }
+
+ public Builder state(final String state) {
+ this.state = state;
+ return this;
+ }
+
+ public Builder annotationData(final String annotationData) {
+ this.annotationData = annotationData;
+ return this;
+ }
+
+ public Builder properties(final Map<String, String> properties) {
+ this.properties = properties;
+ return this;
+ }
+
+ public Builder validateErrors(final Collection<String> validationErrors) {
+ this.validationErrors = validationErrors;
+ return this;
+ }
+
+ public ComponentDetails build() {
+ return new ComponentDetails(this);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/web/ConfigurationAction.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/web/ConfigurationAction.java b/nifi-api/src/main/java/org/apache/nifi/web/ConfigurationAction.java
new file mode 100644
index 0000000..96f2abf
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/web/ConfigurationAction.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web;
+
+/**
+ * An action that represents the configuration of a component.
+ */
+public class ConfigurationAction {
+
+ private final String id;
+ private final String name;
+ private final String type;
+ private final String field;
+ private final String previousValue;
+ private final String value;
+
+ private ConfigurationAction(final Builder builder) {
+ this.id = builder.id;
+ this.name = builder.name;
+ this.type = builder.type;
+ this.field = builder.field;
+ this.previousValue = builder.previousValue;
+ this.value = builder.value;
+ }
+
+ /**
+ * @return id of the component being modified
+ */
+ public String getId() {
+ return id;
+ }
+
+ /**
+ * @return name of the component being modified
+ */
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * @return type of the component being modified
+ */
+ public String getType() {
+ return type;
+ }
+
+ /**
+ * @return the name of the field, property, etc that has been modified
+ */
+ public String getField() {
+ return field;
+ }
+
+ /**
+ * @return the previous value
+ */
+ public String getPreviousValue() {
+ return previousValue;
+ }
+
+ /**
+ * @return the new value
+ */
+ public String getValue() {
+ return value;
+ }
+
+ public static class Builder {
+
+ private String id;
+ private String name;
+ private String type;
+ private String field;
+ private String previousValue;
+ private String value;
+
+ public Builder id(final String id) {
+ this.id = id;
+ return this;
+ }
+
+ public Builder name(final String name) {
+ this.name = name;
+ return this;
+ }
+
+ public Builder type(final String type) {
+ this.type = type;
+ return this;
+ }
+
+ public Builder field(final String field) {
+ this.field = field;
+ return this;
+ }
+
+ public Builder previousValue(final String previousValue) {
+ this.previousValue = previousValue;
+ return this;
+ }
+
+ public Builder value(final String value) {
+ this.value = value;
+ return this;
+ }
+
+ public ConfigurationAction build() {
+ return new ConfigurationAction(this);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/web/InvalidRevisionException.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/web/InvalidRevisionException.java b/nifi-api/src/main/java/org/apache/nifi/web/InvalidRevisionException.java
new file mode 100644
index 0000000..8e04f69
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/web/InvalidRevisionException.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web;
+
+/**
+ * Exception indicating that the client has included an old revision in their
+ * request.
+ */
+@SuppressWarnings("serial")
+public class InvalidRevisionException extends RuntimeException {
+
+ public InvalidRevisionException(String message) {
+ super(message);
+ }
+
+ public InvalidRevisionException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/web/NiFiWebConfigurationContext.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/web/NiFiWebConfigurationContext.java b/nifi-api/src/main/java/org/apache/nifi/web/NiFiWebConfigurationContext.java
new file mode 100644
index 0000000..ae32b10
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/web/NiFiWebConfigurationContext.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web;
+
+import java.util.Collection;
+
+import org.apache.nifi.controller.ControllerService;
+
+/**
+ * NiFi web context providing limited access to dataflow configuration for
+ * component custom UIs.
+ */
+public interface NiFiWebConfigurationContext {
+
+ /**
+ * @param serviceIdentifier of the controller service
+ * @return the ControllerService for the specified identifier. If a
+ * corresponding service cannot be found, null is returned. If this NiFi is
+ * clustered, the only services available will be those those availability
+ * is NCM only
+ */
+ ControllerService getControllerService(String serviceIdentifier);
+
+ /**
+ * Provides a mechanism for custom UIs to save actions to appear in NiFi
+ * configuration history. Note all fields within each Action must be
+ * populated. Null values will result in a failure to insert the audit
+ * record. Since the saving to these actions is separate from the actual
+ * configuration change, a failure to insert here will just generate a
+ * warning log message. The recording of these actions typically happens
+ * after a configuration change is applied. Since those changes have already
+ * been applied to the flow, we cannot revert them because of a failure to
+ * insert an audit record.
+ *
+ * @param requestContext context of the request
+ * @param actions to save
+ * @throws IllegalArgumentException When the requestContext isn't fully
+ * populated or isn't appropriate for the given request
+ */
+ void saveActions(NiFiWebRequestContext requestContext, Collection<ConfigurationAction> actions);
+
+ /**
+ * @return the current user dn. Returns null if no user is found
+ */
+ String getCurrentUserDn();
+
+ /**
+ * @return the current user name. Returns null if no user is found
+ */
+ String getCurrentUserName();
+
+ /**
+ * Sets the annotation data for the underlying component.
+ *
+ * @param configurationContext config context
+ * @param annotationData the data
+ * @return the configuration for the underlying component
+ * @throws ResourceNotFoundException if the underlying component does not
+ * exit
+ * @throws InvalidRevisionException if a revision other than the current
+ * revision is given
+ * @throws ClusterRequestException if the annotation data was unable to be
+ * set for the underlying component. This exception will only be thrown when
+ * operating in a cluster.
+ * @throws IllegalArgumentException When the requestContext isn't fully
+ * populated or isn't appropriate for the given request
+ */
+ ComponentDetails setAnnotationData(NiFiWebConfigurationRequestContext configurationContext, String annotationData)
+ throws ResourceNotFoundException, InvalidRevisionException, ClusterRequestException;
+
+ /**
+ * Gets the details for the underlying component (including configuration,
+ * validation errors, and annotation data).
+ *
+ * @param requestContext context of request
+ * @return the configuration for the underlying component
+ * @throws ResourceNotFoundException if the underlying component does not
+ * exit
+ * @throws ClusterRequestException if the underlying component was unable to
+ * be retrieved from the cluster. This exception will only be thrown when
+ * operating in a cluster.
+ * @throws IllegalArgumentException When the requestContext isn't fully
+ * populated or isn't appropriate for the given request
+ */
+ ComponentDetails getComponentDetails(NiFiWebRequestContext requestContext) throws ResourceNotFoundException, ClusterRequestException;
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/web/NiFiWebConfigurationRequestContext.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/web/NiFiWebConfigurationRequestContext.java b/nifi-api/src/main/java/org/apache/nifi/web/NiFiWebConfigurationRequestContext.java
new file mode 100644
index 0000000..c75d9dc
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/web/NiFiWebConfigurationRequestContext.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web;
+
+/**
+ * Contextual details required to make a configuration request from a UI
+ * extension.
+ */
+public interface NiFiWebConfigurationRequestContext extends NiFiWebRequestContext {
+
+ /**
+ * The revision to include in the request.
+ *
+ * @return the revision
+ */
+ Revision getRevision();
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/web/NiFiWebContext.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/web/NiFiWebContext.java b/nifi-api/src/main/java/org/apache/nifi/web/NiFiWebContext.java
new file mode 100644
index 0000000..55e90e8
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/web/NiFiWebContext.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web;
+
+import java.util.Collection;
+
+import org.apache.nifi.controller.ControllerService;
+
+/**
+ * NiFi web context providing limited access to dataflow configuration for
+ * processor custom UIs.
+ */
+@Deprecated
+public interface NiFiWebContext {
+
+ /**
+ * @param serviceIdentifier identifier of the service
+ * @return the ControllerService for the specified identifier. If a
+ * corresponding service cannot be found, null is returned. If this NiFi is
+ * clustered, the ControllerService is loaded from the NCM
+ */
+ ControllerService getControllerService(String serviceIdentifier);
+
+ /**
+ * Provides a mechanism for custom UIs to save actions to appear in NiFi
+ * configuration history. Note all fields within each Action must be
+ * populated. Null values will result in a failure to insert the audit
+ * record. Since the saving to these actions is separate from the actual
+ * configuration change, a failure to insert here will just generate a
+ * warning log message. The recording of these actions typically happens
+ * after a configuration change is applied. Since those changes have already
+ * been applied to the flow, we cannot revert them because of a failure to
+ * insert an audit record.
+ *
+ * @param actions to save
+ */
+ void saveActions(Collection<ProcessorConfigurationAction> actions);
+
+ /**
+ * @return the current user dn. Returns null if no user is found
+ */
+ String getCurrentUserDn();
+
+ /**
+ * @return the current user name. Returns null if no user is found
+ */
+ String getCurrentUserName();
+
+ /**
+ * Gets the Processor configuration. The given configuration is expected to
+ * contain the following configuration:
+ *
+ * <ul>
+ * <li>revision -- the client identifier and optionally the version
+ * number</li>
+ * <li>processorId -- the id of the processor to retrieve information
+ * for</li>
+ * <li>X509Certificate -- the certificate if this is a secure request</li>
+ * </ul>
+ *
+ * When operating in a clustered environment, if the configuration contains
+ * a X509Certificate, then the certificate information will be forwarded to
+ * the nodes.
+ *
+ * @param config the configuration
+ * @return the processor info object
+ * @throws ResourceNotFoundException if the processor does not exit
+ * @throws ClusterRequestException if the processor was unable to be
+ * retrieved from the cluster. This exception will only be thrown when
+ * operating in a cluster.
+ */
+ ProcessorInfo getProcessor(NiFiWebContextConfig config) throws ResourceNotFoundException, ClusterRequestException;
+
+ /**
+ * Sets the Processor annotation data. The given configuration is expected
+ * to contain the following configuration:
+ *
+ * <ul>
+ * <li>revision -- the client identifier and optionally the version
+ * number</li>
+ * <li>processorId -- the id of the processor to retrieve information
+ * for</li>
+ * <li>X509Certificate -- the certificate if this is a secure request</li>
+ * </ul>
+ *
+ * When operating in a clustered environment, if the configuration contains
+ * a X509Certificate, then the certificate information will be forwarded to
+ * the nodes.
+ *
+ * @param config the configuration
+ * @param annotationData the annotation data
+ * @throws ResourceNotFoundException if the processor does not exit
+ * @throws InvalidRevisionException if a revision other than the current
+ * revision is given
+ * @throws ClusterRequestException if the annotation data was unable to be
+ * set for the processor. This exception will only be thrown when operating
+ * in a cluster.
+ */
+ void setProcessorAnnotationData(NiFiWebContextConfig config, String annotationData)
+ throws ResourceNotFoundException, InvalidRevisionException, ClusterRequestException;
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/web/NiFiWebContextConfig.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/web/NiFiWebContextConfig.java b/nifi-api/src/main/java/org/apache/nifi/web/NiFiWebContextConfig.java
new file mode 100644
index 0000000..2df94e4
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/web/NiFiWebContextConfig.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web;
+
+/**
+ * Context configuration for methods invoked from the NiFiWebContext.
+ */
+@Deprecated
+public interface NiFiWebContextConfig {
+
+ /**
+ * The request protocol scheme (http or https). When scheme is https, the
+ * X509Certificate can be used for subsequent remote requests.
+ *
+ * @return the protocol scheme
+ */
+ String getScheme();
+
+ /**
+ * @return the processor ID
+ */
+ String getProcessorId();
+
+ /**
+ * @return the revision
+ */
+ Revision getRevision();
+
+ /**
+ * Returns the proxied entities chain. The format of the chain is as
+ * follows:
+ *
+ * <code>
+ * <CN=original-proxied-entity><CN=first-proxy><CN=second-proxy>...
+ * </code>
+ *
+ * @return the proxied entities chain or null if no chain
+ */
+ String getProxiedEntitiesChain();
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/web/NiFiWebRequestContext.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/web/NiFiWebRequestContext.java b/nifi-api/src/main/java/org/apache/nifi/web/NiFiWebRequestContext.java
new file mode 100644
index 0000000..9dd44ab
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/web/NiFiWebRequestContext.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web;
+
+/**
+ * Contextual details required to make a request from a UI extension.
+ */
+public interface NiFiWebRequestContext {
+
+ /**
+ * @return the type of UI extension is making the request
+ */
+ UiExtensionType getExtensionType();
+
+ /**
+ * The request protocol scheme (http or https). When scheme is https, the
+ * X509Certificate can be used for subsequent remote requests.
+ *
+ * @return the protocol scheme
+ */
+ String getScheme();
+
+ /**
+ * The id of the component.
+ *
+ * @return the ID
+ */
+ String getId();
+
+ /**
+ * Returns the proxied entities chain. The format of the chain is as
+ * follows:
+ *
+ * <code>
+ * <CN=original-proxied-entity><CN=first-proxy><CN=second-proxy>...
+ * </code>
+ *
+ * @return the proxied entities chain or null if no chain
+ */
+ String getProxiedEntitiesChain();
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/web/ProcessorConfigurationAction.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/web/ProcessorConfigurationAction.java b/nifi-api/src/main/java/org/apache/nifi/web/ProcessorConfigurationAction.java
new file mode 100644
index 0000000..f42063f
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/web/ProcessorConfigurationAction.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web;
+
+/**
+ *
+ */
+@Deprecated
+public class ProcessorConfigurationAction {
+
+ private final String processorId;
+ private final String processorName;
+ private final String processorType;
+ private final String name;
+ private final String previousValue;
+ private final String value;
+
+ private ProcessorConfigurationAction(final Builder builder) {
+ this.processorId = builder.processorId;
+ this.processorName = builder.processorName;
+ this.processorType = builder.processorType;
+ this.name = builder.name;
+ this.previousValue = builder.previousValue;
+ this.value = builder.value;
+ }
+
+ /**
+ * @return the id of the processor
+ */
+ public String getProcessorId() {
+ return processorId;
+ }
+
+ /**
+ * @return the name of the processor being modified
+ */
+ public String getProcessorName() {
+ return processorName;
+ }
+
+ /**
+ * @return the type of the processor being modified
+ */
+ public String getProcessorType() {
+ return processorType;
+ }
+
+ /**
+ * @return the name of the field, property, etc that has been modified.
+ */
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * @return the previous value
+ */
+ public String getPreviousValue() {
+ return previousValue;
+ }
+
+ /**
+ * @return the new value
+ */
+ public String getValue() {
+ return value;
+ }
+
+ public static class Builder {
+
+ private String processorId;
+ private String processorName;
+ private String processorType;
+ private String name;
+ private String previousValue;
+ private String value;
+
+ public Builder processorId(final String processorId) {
+ this.processorId = processorId;
+ return this;
+ }
+
+ public Builder processorName(final String processorName) {
+ this.processorName = processorName;
+ return this;
+ }
+
+ public Builder processorType(final String processorType) {
+ this.processorType = processorType;
+ return this;
+ }
+
+ public Builder name(final String name) {
+ this.name = name;
+ return this;
+ }
+
+ public Builder previousValue(final String previousValue) {
+ this.previousValue = previousValue;
+ return this;
+ }
+
+ public Builder value(final String value) {
+ this.value = value;
+ return this;
+ }
+
+ public ProcessorConfigurationAction build() {
+ return new ProcessorConfigurationAction(this);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/web/ProcessorInfo.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/web/ProcessorInfo.java b/nifi-api/src/main/java/org/apache/nifi/web/ProcessorInfo.java
new file mode 100644
index 0000000..e87e73e
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/web/ProcessorInfo.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web;
+
+import java.util.Collection;
+import java.util.Map;
+
+/**
+ *
+ */
+@Deprecated
+public class ProcessorInfo {
+
+ private final String id;
+ private final String name;
+ private final String state;
+ private final String annotationData;
+ private final Map<String, String> properties;
+ private final Collection<String> validationErrors;
+
+ private ProcessorInfo(final Builder builder) {
+ this.id = builder.id;
+ this.name = builder.name;
+ this.state = builder.state;
+ this.annotationData = builder.annotationData;
+ this.properties = builder.properties;
+ this.validationErrors = builder.validationErrors;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String getState() {
+ return state;
+ }
+
+ public String getAnnotationData() {
+ return annotationData;
+ }
+
+ public Map<String, String> getProperties() {
+ return properties;
+ }
+
+ public Collection<String> getValidationErrors() {
+ return validationErrors;
+ }
+
+ public static final class Builder {
+
+ private String id;
+ private String name;
+ private String state;
+ private String annotationData;
+ private Map<String, String> properties;
+ private Collection<String> validationErrors;
+
+ public Builder id(final String id) {
+ this.id = id;
+ return this;
+ }
+
+ public Builder name(final String name) {
+ this.name = name;
+ return this;
+ }
+
+ public Builder state(final String state) {
+ this.state = state;
+ return this;
+ }
+
+ public Builder annotationData(final String annotationData) {
+ this.annotationData = annotationData;
+ return this;
+ }
+
+ public Builder properties(final Map<String, String> properties) {
+ this.properties = properties;
+ return this;
+ }
+
+ public Builder validateErrors(final Collection<String> validationErrors) {
+ this.validationErrors = validationErrors;
+ return this;
+ }
+
+ public ProcessorInfo build() {
+ return new ProcessorInfo(this);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/web/ResourceNotFoundException.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/web/ResourceNotFoundException.java b/nifi-api/src/main/java/org/apache/nifi/web/ResourceNotFoundException.java
new file mode 100644
index 0000000..b2f2e57
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/web/ResourceNotFoundException.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web;
+
+/**
+ * Exception indicating that the desired resource was not found.
+ */
+@SuppressWarnings("serial")
+public class ResourceNotFoundException extends RuntimeException {
+
+ public ResourceNotFoundException(String message) {
+ super(message);
+ }
+
+ public ResourceNotFoundException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/web/Revision.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/web/Revision.java b/nifi-api/src/main/java/org/apache/nifi/web/Revision.java
new file mode 100644
index 0000000..a4d36d2
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/web/Revision.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web;
+
+import java.io.Serializable;
+
+/**
+ * A model object representing a revision. Equality is defined as either a
+ * matching version number or matching non-empty client IDs.
+ *
+ * @Immutable
+ * @Threadsafe
+ */
+public class Revision implements Serializable {
+
+ /**
+ * the version number
+ */
+ private final Long version;
+
+ /**
+ * the client ID
+ */
+ private final String clientId;
+
+ public Revision(Long revision, String clientId) {
+ this.version = revision;
+ this.clientId = clientId;
+ }
+
+ public String getClientId() {
+ return clientId;
+ }
+
+ public Long getVersion() {
+ return version;
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+
+ if ((obj instanceof Revision) == false) {
+ return false;
+ }
+
+ Revision thatRevision = (Revision) obj;
+ if (this.version != null && this.version.equals(thatRevision.version)) {
+ return true;
+ } else {
+ return clientId != null && !clientId.trim().isEmpty() && clientId.equals(thatRevision.getClientId());
+ }
+
+ }
+
+ @Override
+ public int hashCode() {
+ int hash = 5;
+ hash = 59 * hash + (this.version != null ? this.version.hashCode() : 0);
+ hash = 59 * hash + (this.clientId != null ? this.clientId.hashCode() : 0);
+ return hash;
+ }
+
+ @Override
+ public String toString() {
+ return "[" + version + ", " + clientId + ']';
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/web/UiExtensionType.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/web/UiExtensionType.java b/nifi-api/src/main/java/org/apache/nifi/web/UiExtensionType.java
new file mode 100644
index 0000000..e3b0f8a
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/web/UiExtensionType.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web;
+
+/**
+ * Types of UI extensions. Since a UI extension could support multiple types of
+ * custom UIs it will need to include the type so the framework can appropriate
+ * understand and process the request (recording actions in the audit database,
+ * replicating a request throughout the cluster to the appropriate endpoints,
+ * etc).
+ */
+public enum UiExtensionType {
+
+ ContentViewer,
+ ProcessorConfiguration,
+ ControllerServiceConfiguration,
+ ReportingTaskConfiguration
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/web/ViewableContent.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/web/ViewableContent.java b/nifi-api/src/main/java/org/apache/nifi/web/ViewableContent.java
new file mode 100644
index 0000000..180385e
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/web/ViewableContent.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * Interface for obtaining content from the NiFi content repository.
+ */
+public interface ViewableContent {
+
+ public static final String CONTENT_REQUEST_ATTRIBUTE = "org.apache.nifi.web.content";
+
+ public enum DisplayMode {
+
+ Original,
+ Formatted,
+ Hex;
+ }
+
+ /**
+ * @return stream to the viewable content. The data stream can only be read once
+ * so an extension can call this method or getContent
+ */
+ InputStream getContentStream();
+
+ /**
+ * @return the content as a string. The data stream can only be read once so an
+ * extension can call this method or getContentStream
+ * @throws java.io.IOException if unable to read content
+ */
+ String getContent() throws IOException;
+
+ /**
+ * @return the desired display mode. If the mode is Hex the framework will
+ * handle generating the mark up. The only values that an extension will see
+ * is Original or Formatted
+ */
+ DisplayMode getDisplayMode();
+
+ /**
+ * @return contents file name
+ */
+ String getFileName();
+
+ /**
+ * @return mime type of the content
+ */
+ String getContentType();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/test/java/org/apache/nifi/components/TestPropertyDescriptor.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/test/java/org/apache/nifi/components/TestPropertyDescriptor.java b/nifi-api/src/test/java/org/apache/nifi/components/TestPropertyDescriptor.java
new file mode 100644
index 0000000..e3043be
--- /dev/null
+++ b/nifi-api/src/test/java/org/apache/nifi/components/TestPropertyDescriptor.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.components;
+
+import static org.junit.Assert.assertNotNull;
+
+import org.apache.nifi.components.PropertyDescriptor.Builder;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+/**
+ * Regression test for issue NIFI-49, to ensure that if a Processor's Property's
+ * Default Value is not allowed, the Exception thrown should indicate what the
+ * default value is
+ */
+public class TestPropertyDescriptor {
+
+ private static Builder invalidDescriptorBuilder;
+ private static Builder validDescriptorBuilder;
+ private static String DEFAULT_VALUE = "Default Value";
+
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ @BeforeClass
+ public static void setUp() {
+ validDescriptorBuilder = new PropertyDescriptor.Builder().name("").allowableValues("Allowable Value", "Another Allowable Value").defaultValue("Allowable Value");
+ invalidDescriptorBuilder = new PropertyDescriptor.Builder().name("").allowableValues("Allowable Value", "Another Allowable Value").defaultValue(DEFAULT_VALUE);
+ }
+
+ @Test
+ public void testExceptionThrownByDescriptorWithInvalidDefaultValue() {
+ thrown.expect(IllegalStateException.class);
+ thrown.expectMessage("[" + DEFAULT_VALUE + "]");
+
+ invalidDescriptorBuilder.build();
+ }
+
+ @Test
+ public void testNoExceptionThrownByPropertyDescriptorWithValidDefaultValue() {
+ assertNotNull(validDescriptorBuilder.build());
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/test/java/org/apache/nifi/processor/TestDataUnit.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/test/java/org/apache/nifi/processor/TestDataUnit.java b/nifi-api/src/test/java/org/apache/nifi/processor/TestDataUnit.java
new file mode 100644
index 0000000..3e6a235
--- /dev/null
+++ b/nifi-api/src/test/java/org/apache/nifi/processor/TestDataUnit.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ *
+ */
+public class TestDataUnit {
+
+ @Test
+ public void testParseWithIntegerValue() {
+ assertEquals(300L, DataUnit.parseDataSize("300 B", DataUnit.B).longValue());
+ assertEquals(300L * 1024L, DataUnit.parseDataSize("300 KB", DataUnit.B).longValue());
+ assertEquals(300L * 1024L * 1024L, DataUnit.parseDataSize("300 MB", DataUnit.B).longValue());
+ assertEquals(300L * 1024L * 1024L * 1024L, DataUnit.parseDataSize("300 GB", DataUnit.B).longValue());
+ }
+
+ @Test
+ public void testParseWithDecimalValue() {
+ assertEquals(300L, DataUnit.parseDataSize("300 B", DataUnit.B).longValue());
+ assertEquals((long) (3.22D * 1024D), DataUnit.parseDataSize("3.22 KB", DataUnit.B).longValue());
+ assertEquals((long) (3.22D * 1024D * 1024D), DataUnit.parseDataSize("3.22 MB", DataUnit.B).longValue());
+ assertEquals((long) (3.22D * 1024D * 1024D * 1024D), DataUnit.parseDataSize("3.22 GB", DataUnit.B).longValue());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/test/resources/logback-test.xml
----------------------------------------------------------------------
diff --git a/nifi-api/src/test/resources/logback-test.xml b/nifi-api/src/test/resources/logback-test.xml
new file mode 100644
index 0000000..162b71b
--- /dev/null
+++ b/nifi-api/src/test/resources/logback-test.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+ <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+ <pattern>%-4r [%t] %-5p %c - %m%n</pattern>
+ </encoder>
+ </appender>
+ <logger name="org.apache.nifi.processor" level="DEBUG"/>
+ <root level ="DEBUG">
+ <appender-ref ref="CONSOLE"/>
+ </root>
+</configuration>
[40/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/RunNiFi.java
----------------------------------------------------------------------
diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/RunNiFi.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/RunNiFi.java
new file mode 100644
index 0000000..a48a1de
--- /dev/null
+++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/RunNiFi.java
@@ -0,0 +1,1008 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.bootstrap;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.Reader;
+import java.lang.reflect.Field;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.attribute.PosixFilePermission;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ * <p>
+ * The class which bootstraps Apache NiFi. This class looks for the
+ * bootstrap.conf file by looking in the following places (in order):</p>
+ * <ol>
+ * <li>Java System Property named
+ * {@code org.apache.nifi.bootstrap.config.file}</li>
+ * <li>${NIFI_HOME}/./conf/bootstrap.conf, where ${NIFI_HOME} references an
+ * environment variable {@code NIFI_HOME}</li>
+ * <li>./conf/bootstrap.conf, where {@code ./} represents the working
+ * directory.</li>
+ * </ol>
+ *
+ * If the {@code bootstrap.conf} file cannot be found, throws a {@code FileNotFoundException}.
+ */
+public class RunNiFi {
+
+ public static final String DEFAULT_CONFIG_FILE = "./conf/bootstrap.conf";
+ public static final String DEFAULT_NIFI_PROPS_FILE = "./conf/nifi.properties";
+ public static final String DEFAULT_JAVA_CMD = "java";
+
+ public static final String GRACEFUL_SHUTDOWN_PROP = "graceful.shutdown.seconds";
+ public static final String DEFAULT_GRACEFUL_SHUTDOWN_VALUE = "20";
+
+ public static final String RUN_AS_PROP = "run.as";
+
+ public static final int MAX_RESTART_ATTEMPTS = 5;
+ public static final int STARTUP_WAIT_SECONDS = 60;
+
+ public static final String SHUTDOWN_CMD = "SHUTDOWN";
+ public static final String PING_CMD = "PING";
+ public static final String DUMP_CMD = "DUMP";
+
+ private volatile boolean autoRestartNiFi = true;
+ private volatile int ccPort = -1;
+ private volatile long nifiPid = -1L;
+ private volatile String secretKey;
+ private volatile ShutdownHook shutdownHook;
+
+ private final Lock lock = new ReentrantLock();
+ private final Condition startupCondition = lock.newCondition();
+
+ private final File bootstrapConfigFile;
+
+ // used for logging initial info; these will be logged to console by default when the app is started
+ private final Logger cmdLogger = LoggerFactory.getLogger("org.apache.nifi.bootstrap.Command");
+ // used for logging all info. These by default will be written to the log file
+ private final Logger defaultLogger = LoggerFactory.getLogger(RunNiFi.class);
+
+
+ private final ExecutorService loggingExecutor;
+ private volatile Set<Future<?>> loggingFutures = new HashSet<>(2);
+
+ public RunNiFi(final File bootstrapConfigFile, final boolean verbose) {
+ this.bootstrapConfigFile = bootstrapConfigFile;
+
+ loggingExecutor = Executors.newFixedThreadPool(2, new ThreadFactory() {
+ @Override
+ public Thread newThread(final Runnable runnable) {
+ final Thread t = Executors.defaultThreadFactory().newThread(runnable);
+ t.setDaemon(true);
+ t.setName("NiFi logging handler");
+ return t;
+ }
+ });
+ }
+
+ private static void printUsage() {
+ System.out.println("Usage:");
+ System.out.println();
+ System.out.println("java org.apache.nifi.bootstrap.RunNiFi [<-verbose>] <command> [options]");
+ System.out.println();
+ System.out.println("Valid commands include:");
+ System.out.println("");
+ System.out.println("Start : Start a new instance of Apache NiFi");
+ System.out.println("Stop : Stop a running instance of Apache NiFi");
+ System.out.println("Restart : Stop Apache NiFi, if it is running, and then start a new instance");
+ System.out.println("Status : Determine if there is a running instance of Apache NiFi");
+ System.out.println("Dump : Write a Thread Dump to the file specified by [options], or to the log if no file is given");
+ System.out.println("Run : Start a new instance of Apache NiFi and monitor the Process, restarting if the instance dies");
+ System.out.println();
+ }
+
+ private static String[] shift(final String[] orig) {
+ return Arrays.copyOfRange(orig, 1, orig.length);
+ }
+
+ public static void main(String[] args) throws IOException, InterruptedException {
+ if (args.length < 1 || args.length > 3) {
+ printUsage();
+ return;
+ }
+
+ File dumpFile = null;
+ boolean verbose = false;
+ if (args[0].equals("-verbose")) {
+ verbose = true;
+ args = shift(args);
+ }
+
+ final String cmd = args[0];
+ if (cmd.equals("dump")) {
+ if (args.length > 1) {
+ dumpFile = new File(args[1]);
+ } else {
+ dumpFile = null;
+ }
+ }
+
+ switch (cmd.toLowerCase()) {
+ case "start":
+ case "run":
+ case "stop":
+ case "status":
+ case "dump":
+ case "restart":
+ break;
+ default:
+ printUsage();
+ return;
+ }
+
+ String configFilename = System.getProperty("org.apache.nifi.bootstrap.config.file");
+
+ if (configFilename == null) {
+ final String nifiHome = System.getenv("NIFI_HOME");
+ if (nifiHome != null) {
+ final File nifiHomeFile = new File(nifiHome.trim());
+ final File configFile = new File(nifiHomeFile, DEFAULT_CONFIG_FILE);
+ configFilename = configFile.getAbsolutePath();
+ }
+ }
+
+ if (configFilename == null) {
+ configFilename = DEFAULT_CONFIG_FILE;
+ }
+
+ final File configFile = new File(configFilename);
+
+ final RunNiFi runNiFi = new RunNiFi(configFile, verbose);
+
+ switch (cmd.toLowerCase()) {
+ case "start":
+ runNiFi.start();
+ break;
+ case "run":
+ runNiFi.start();
+ break;
+ case "stop":
+ runNiFi.stop();
+ break;
+ case "status":
+ runNiFi.status();
+ break;
+ case "restart":
+ runNiFi.stop();
+ runNiFi.start();
+ break;
+ case "dump":
+ runNiFi.dump(dumpFile);
+ break;
+ }
+ }
+
+ File getStatusFile() {
+ return getStatusFile(defaultLogger);
+ }
+
+ public File getStatusFile(final Logger logger) {
+ final File confDir = bootstrapConfigFile.getParentFile();
+ final File nifiHome = confDir.getParentFile();
+ final File bin = new File(nifiHome, "bin");
+ final File statusFile = new File(bin, "nifi.pid");
+
+ logger.debug("Status File: {}", statusFile);
+
+ return statusFile;
+ }
+
+ private Properties loadProperties(final Logger logger) throws IOException {
+ final Properties props = new Properties();
+ final File statusFile = getStatusFile(logger);
+ if (statusFile == null || !statusFile.exists()) {
+ logger.debug("No status file to load properties from");
+ return props;
+ }
+
+ try (final FileInputStream fis = new FileInputStream(getStatusFile(logger))) {
+ props.load(fis);
+ }
+
+ final Map<Object, Object> modified = new HashMap<>(props);
+ modified.remove("secret.key");
+ logger.debug("Properties: {}", modified);
+
+ return props;
+ }
+
+ private synchronized void saveProperties(final Properties nifiProps, final Logger logger) throws IOException {
+ final File statusFile = getStatusFile(logger);
+ if (statusFile.exists() && !statusFile.delete()) {
+ logger.warn("Failed to delete {}", statusFile);
+ }
+
+ if (!statusFile.createNewFile()) {
+ throw new IOException("Failed to create file " + statusFile);
+ }
+
+ try {
+ final Set<PosixFilePermission> perms = new HashSet<>();
+ perms.add(PosixFilePermission.OWNER_READ);
+ perms.add(PosixFilePermission.OWNER_WRITE);
+ Files.setPosixFilePermissions(statusFile.toPath(), perms);
+ } catch (final Exception e) {
+ logger.warn("Failed to set permissions so that only the owner can read status file {}; "
+ + "this may allows others to have access to the key needed to communicate with NiFi. "
+ + "Permissions should be changed so that only the owner can read this file", statusFile);
+ }
+
+ try (final FileOutputStream fos = new FileOutputStream(statusFile)) {
+ nifiProps.store(fos, null);
+ fos.getFD().sync();
+ }
+
+ logger.debug("Saved Properties {} to {}", new Object[]{nifiProps, statusFile});
+ }
+
+ private boolean isPingSuccessful(final int port, final String secretKey, final Logger logger) {
+ logger.debug("Pinging {}", port);
+
+ try (final Socket socket = new Socket("localhost", port)) {
+ final OutputStream out = socket.getOutputStream();
+ out.write((PING_CMD + " " + secretKey + "\n").getBytes(StandardCharsets.UTF_8));
+ out.flush();
+
+ logger.debug("Sent PING command");
+ socket.setSoTimeout(5000);
+ final InputStream in = socket.getInputStream();
+ final BufferedReader reader = new BufferedReader(new InputStreamReader(in));
+ final String response = reader.readLine();
+ logger.debug("PING response: {}", response);
+ out.close();
+ reader.close();
+
+ return PING_CMD.equals(response);
+ } catch (final IOException ioe) {
+ return false;
+ }
+ }
+
+ private Integer getCurrentPort(final Logger logger) throws IOException {
+ final Properties props = loadProperties(logger);
+ final String portVal = props.getProperty("port");
+ if (portVal == null) {
+ logger.debug("No Port found in status file");
+ return null;
+ } else {
+ logger.debug("Port defined in status file: {}", portVal);
+ }
+
+ final int port = Integer.parseInt(portVal);
+ final boolean success = isPingSuccessful(port, props.getProperty("secret.key"), logger);
+ if (success) {
+ logger.debug("Successful PING on port {}", port);
+ return port;
+ }
+
+ final String pid = props.getProperty("pid");
+ logger.debug("PID in status file is {}", pid);
+ if (pid != null) {
+ final boolean procRunning = isProcessRunning(pid, logger);
+ if (procRunning) {
+ return port;
+ } else {
+ return null;
+ }
+ }
+
+ return null;
+ }
+
+ private boolean isProcessRunning(final String pid, final Logger logger) {
+ try {
+ // We use the "ps" command to check if the process is still running.
+ final ProcessBuilder builder = new ProcessBuilder();
+
+ builder.command("ps", "-p", pid);
+ final Process proc = builder.start();
+
+ // Look for the pid in the output of the 'ps' command.
+ boolean running = false;
+ String line;
+ try (final InputStream in = proc.getInputStream();
+ final Reader streamReader = new InputStreamReader(in);
+ final BufferedReader reader = new BufferedReader(streamReader)) {
+
+ while ((line = reader.readLine()) != null) {
+ if (line.trim().startsWith(pid)) {
+ running = true;
+ }
+ }
+ }
+
+ // If output of the ps command had our PID, the process is running.
+ if (running) {
+ logger.debug("Process with PID {} is running", pid);
+ } else {
+ logger.debug("Process with PID {} is not running", pid);
+ }
+
+ return running;
+ } catch (final IOException ioe) {
+ System.err.println("Failed to determine if Process " + pid + " is running; assuming that it is not");
+ return false;
+ }
+ }
+
+ private Status getStatus(final Logger logger) {
+ final Properties props;
+ try {
+ props = loadProperties(logger);
+ } catch (final IOException ioe) {
+ return new Status(null, null, false, false);
+ }
+
+ if (props == null) {
+ return new Status(null, null, false, false);
+ }
+
+ final String portValue = props.getProperty("port");
+ final String pid = props.getProperty("pid");
+ final String secretKey = props.getProperty("secret.key");
+
+ if (portValue == null && pid == null) {
+ return new Status(null, null, false, false);
+ }
+
+ Integer port = null;
+ boolean pingSuccess = false;
+ if (portValue != null) {
+ try {
+ port = Integer.parseInt(portValue);
+ pingSuccess = isPingSuccessful(port, secretKey, logger);
+ } catch (final NumberFormatException nfe) {
+ return new Status(null, null, false, false);
+ }
+ }
+
+ if (pingSuccess) {
+ return new Status(port, pid, true, true);
+ }
+
+ final boolean alive = (pid == null) ? false : isProcessRunning(pid, logger);
+ return new Status(port, pid, pingSuccess, alive);
+ }
+
+ public void status() throws IOException {
+ final Logger logger = cmdLogger;
+ final Status status = getStatus(logger);
+ if (status.isRespondingToPing()) {
+ logger.info("Apache NiFi is currently running, listening to Bootstrap on port {}, PID={}",
+ new Object[]{status.getPort(), status.getPid() == null ? "unknkown" : status.getPid()});
+ return;
+ }
+
+ if (status.isProcessRunning()) {
+ logger.info("Apache NiFi is running at PID {} but is not responding to ping requests", status.getPid());
+ return;
+ }
+
+ if (status.getPort() == null) {
+ logger.info("Apache NiFi is not running");
+ return;
+ }
+
+ if (status.getPid() == null) {
+ logger.info("Apache NiFi is not responding to Ping requests. The process may have died or may be hung");
+ } else {
+ logger.info("Apache NiFi is not running");
+ }
+ }
+
+ /**
+ * Writes a NiFi thread dump to the given file; if file is null, logs at
+ * INFO level instead.
+ *
+ * @param dumpFile the file to write the dump content to
+ * @throws IOException if any issues occur while writing the dump file
+ */
+ public void dump(final File dumpFile) throws IOException {
+ final Logger logger = defaultLogger; // dump to bootstrap log file by default
+ final Integer port = getCurrentPort(logger);
+ if (port == null) {
+ logger.info("Apache NiFi is not currently running");
+ return;
+ }
+
+ final Properties nifiProps = loadProperties(logger);
+ final String secretKey = nifiProps.getProperty("secret.key");
+
+ final StringBuilder sb = new StringBuilder();
+ try (final Socket socket = new Socket()) {
+ logger.debug("Connecting to NiFi instance");
+ socket.setSoTimeout(60000);
+ socket.connect(new InetSocketAddress("localhost", port));
+ logger.debug("Established connection to NiFi instance.");
+ socket.setSoTimeout(60000);
+
+ logger.debug("Sending DUMP Command to port {}", port);
+ final OutputStream out = socket.getOutputStream();
+ out.write((DUMP_CMD + " " + secretKey + "\n").getBytes(StandardCharsets.UTF_8));
+ out.flush();
+
+ final InputStream in = socket.getInputStream();
+ try (final BufferedReader reader = new BufferedReader(new InputStreamReader(in))) {
+ String line;
+ while ((line = reader.readLine()) != null) {
+ sb.append(line).append("\n");
+ }
+ }
+ }
+
+ final String dump = sb.toString();
+ if (dumpFile == null) {
+ logger.info(dump);
+ } else {
+ try (final FileOutputStream fos = new FileOutputStream(dumpFile)) {
+ fos.write(dump.getBytes(StandardCharsets.UTF_8));
+ }
+ // we want to log to the console (by default) that we wrote the thread dump to the specified file
+ cmdLogger.info("Successfully wrote thread dump to {}", dumpFile.getAbsolutePath());
+ }
+ }
+
+ public void stop() throws IOException {
+ final Logger logger = cmdLogger;
+ final Integer port = getCurrentPort(logger);
+ if (port == null) {
+ logger.info("Apache NiFi is not currently running");
+ return;
+ }
+
+ final Properties nifiProps = loadProperties(logger);
+ final String secretKey = nifiProps.getProperty("secret.key");
+
+ final File statusFile = getStatusFile(logger);
+ if (statusFile.exists() && !statusFile.delete()) {
+ logger.error("Failed to delete status file {}; this file should be cleaned up manually", statusFile);
+ }
+
+ try (final Socket socket = new Socket()) {
+ logger.debug("Connecting to NiFi instance");
+ socket.setSoTimeout(60000);
+ socket.connect(new InetSocketAddress("localhost", port));
+ logger.debug("Established connection to NiFi instance.");
+ socket.setSoTimeout(60000);
+
+ logger.debug("Sending SHUTDOWN Command to port {}", port);
+ final OutputStream out = socket.getOutputStream();
+ out.write((SHUTDOWN_CMD + " " + secretKey + "\n").getBytes(StandardCharsets.UTF_8));
+ out.flush();
+ socket.shutdownOutput();
+
+ final InputStream in = socket.getInputStream();
+ int lastChar;
+ final StringBuilder sb = new StringBuilder();
+ while ((lastChar = in.read()) > -1) {
+ sb.append((char) lastChar);
+ }
+ final String response = sb.toString().trim();
+
+ logger.debug("Received response to SHUTDOWN command: {}", response);
+
+ if (SHUTDOWN_CMD.equals(response)) {
+ logger.info("Apache NiFi has accepted the Shutdown Command and is shutting down now");
+
+ final String pid = nifiProps.getProperty("pid");
+ if (pid != null) {
+ final Properties bootstrapProperties = new Properties();
+ try (final FileInputStream fis = new FileInputStream(bootstrapConfigFile)) {
+ bootstrapProperties.load(fis);
+ }
+
+ String gracefulShutdown = bootstrapProperties.getProperty(GRACEFUL_SHUTDOWN_PROP, DEFAULT_GRACEFUL_SHUTDOWN_VALUE);
+ int gracefulShutdownSeconds;
+ try {
+ gracefulShutdownSeconds = Integer.parseInt(gracefulShutdown);
+ } catch (final NumberFormatException nfe) {
+ gracefulShutdownSeconds = Integer.parseInt(DEFAULT_GRACEFUL_SHUTDOWN_VALUE);
+ }
+
+ final long startWait = System.nanoTime();
+ while (isProcessRunning(pid, logger)) {
+ logger.info("Waiting for Apache NiFi to finish shutting down...");
+ final long waitNanos = System.nanoTime() - startWait;
+ final long waitSeconds = TimeUnit.NANOSECONDS.toSeconds(waitNanos);
+ if (waitSeconds >= gracefulShutdownSeconds && gracefulShutdownSeconds > 0) {
+ if (isProcessRunning(pid, logger)) {
+ logger.warn("NiFi has not finished shutting down after {} seconds. Killing process.", gracefulShutdownSeconds);
+ try {
+ killProcessTree(pid, logger);
+ } catch (final IOException ioe) {
+ logger.error("Failed to kill Process with PID {}", pid);
+ }
+ }
+ break;
+ } else {
+ try {
+ Thread.sleep(2000L);
+ } catch (final InterruptedException ie) {
+ }
+ }
+ }
+
+ logger.info("NiFi has finished shutting down.");
+ }
+ } else {
+ logger.error("When sending SHUTDOWN command to NiFi, got unexpected response {}", response);
+ }
+ } catch (final IOException ioe) {
+ logger.error("Failed to send shutdown command to port {} due to {}", new Object[]{port, ioe.toString(), ioe});
+ }
+ }
+
+ private static List<String> getChildProcesses(final String ppid) throws IOException {
+ final Process proc = Runtime.getRuntime().exec(new String[]{"ps", "-o", "pid", "--no-headers", "--ppid", ppid});
+ final List<String> childPids = new ArrayList<>();
+ try (final InputStream in = proc.getInputStream();
+ final BufferedReader reader = new BufferedReader(new InputStreamReader(in))) {
+
+ String line;
+ while ((line = reader.readLine()) != null) {
+ childPids.add(line.trim());
+ }
+ }
+
+ return childPids;
+ }
+
+ private void killProcessTree(final String pid, final Logger logger) throws IOException {
+ logger.debug("Killing Process Tree for PID {}", pid);
+
+ final List<String> children = getChildProcesses(pid);
+ logger.debug("Children of PID {}: {}", new Object[]{pid, children});
+
+ for (final String childPid : children) {
+ killProcessTree(childPid, logger);
+ }
+
+ Runtime.getRuntime().exec(new String[]{"kill", "-9", pid});
+ }
+
+ public static boolean isAlive(final Process process) {
+ try {
+ process.exitValue();
+ return false;
+ } catch (final IllegalStateException | IllegalThreadStateException itse) {
+ return true;
+ }
+ }
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public void start() throws IOException, InterruptedException {
+ final Integer port = getCurrentPort(cmdLogger);
+ if (port != null) {
+ cmdLogger.info("Apache NiFi is already running, listening to Bootstrap on port " + port);
+ return;
+ }
+
+ final ProcessBuilder builder = new ProcessBuilder();
+
+ if (!bootstrapConfigFile.exists()) {
+ throw new FileNotFoundException(bootstrapConfigFile.getAbsolutePath());
+ }
+
+ final Properties properties = new Properties();
+ try (final FileInputStream fis = new FileInputStream(bootstrapConfigFile)) {
+ properties.load(fis);
+ }
+
+ final Map<String, String> props = new HashMap<>();
+ props.putAll((Map) properties);
+
+ final String specifiedWorkingDir = props.get("working.dir");
+ if (specifiedWorkingDir != null) {
+ builder.directory(new File(specifiedWorkingDir));
+ }
+
+ final File bootstrapConfigAbsoluteFile = bootstrapConfigFile.getAbsoluteFile();
+ final File binDir = bootstrapConfigAbsoluteFile.getParentFile();
+ final File workingDir = binDir.getParentFile();
+
+ if (specifiedWorkingDir == null) {
+ builder.directory(workingDir);
+ }
+
+ final String libFilename = replaceNull(props.get("lib.dir"), "./lib").trim();
+ File libDir = getFile(libFilename, workingDir);
+
+ final String confFilename = replaceNull(props.get("conf.dir"), "./conf").trim();
+ File confDir = getFile(confFilename, workingDir);
+
+ String nifiPropsFilename = props.get("props.file");
+ if (nifiPropsFilename == null) {
+ if (confDir.exists()) {
+ nifiPropsFilename = new File(confDir, "nifi.properties").getAbsolutePath();
+ } else {
+ nifiPropsFilename = DEFAULT_CONFIG_FILE;
+ }
+ }
+
+ nifiPropsFilename = nifiPropsFilename.trim();
+
+ final List<String> javaAdditionalArgs = new ArrayList<>();
+ for (final Map.Entry<String, String> entry : props.entrySet()) {
+ final String key = entry.getKey();
+ final String value = entry.getValue();
+
+ if (key.startsWith("java.arg")) {
+ javaAdditionalArgs.add(value);
+ }
+ }
+
+ final File[] libFiles = libDir.listFiles(new FilenameFilter() {
+ @Override
+ public boolean accept(final File dir, final String filename) {
+ return filename.toLowerCase().endsWith(".jar");
+ }
+ });
+
+ if (libFiles == null || libFiles.length == 0) {
+ throw new RuntimeException("Could not find lib directory at " + libDir.getAbsolutePath());
+ }
+
+ final File[] confFiles = confDir.listFiles();
+ if (confFiles == null || confFiles.length == 0) {
+ throw new RuntimeException("Could not find conf directory at " + confDir.getAbsolutePath());
+ }
+
+ final List<String> cpFiles = new ArrayList<>(confFiles.length + libFiles.length);
+ cpFiles.add(confDir.getAbsolutePath());
+ for (final File file : libFiles) {
+ cpFiles.add(file.getAbsolutePath());
+ }
+
+ final StringBuilder classPathBuilder = new StringBuilder();
+ for (int i = 0; i < cpFiles.size(); i++) {
+ final String filename = cpFiles.get(i);
+ classPathBuilder.append(filename);
+ if (i < cpFiles.size() - 1) {
+ classPathBuilder.append(File.pathSeparatorChar);
+ }
+ }
+
+ final String classPath = classPathBuilder.toString();
+ String javaCmd = props.get("java");
+ if (javaCmd == null) {
+ javaCmd = DEFAULT_JAVA_CMD;
+ }
+ if (javaCmd.equals(DEFAULT_JAVA_CMD)) {
+ String javaHome = System.getenv("JAVA_HOME");
+ if (javaHome != null) {
+ String fileExtension = isWindows() ? ".exe" : "";
+ File javaFile = new File(javaHome + File.separatorChar + "bin"
+ + File.separatorChar + "java" + fileExtension);
+ if (javaFile.exists() && javaFile.canExecute()) {
+ javaCmd = javaFile.getAbsolutePath();
+ }
+ }
+ }
+
+ final NiFiListener listener = new NiFiListener();
+ final int listenPort = listener.start(this);
+
+ final List<String> cmd = new ArrayList<>();
+
+ cmd.add(javaCmd);
+ cmd.add("-classpath");
+ cmd.add(classPath);
+ cmd.addAll(javaAdditionalArgs);
+ cmd.add("-Dnifi.properties.file.path=" + nifiPropsFilename);
+ cmd.add("-Dnifi.bootstrap.listen.port=" + listenPort);
+ cmd.add("-Dapp=NiFi");
+ cmd.add("org.apache.nifi.NiFi");
+
+ builder.command(cmd);
+
+ final StringBuilder cmdBuilder = new StringBuilder();
+ for (final String s : cmd) {
+ cmdBuilder.append(s).append(" ");
+ }
+
+ cmdLogger.info("Starting Apache NiFi...");
+ cmdLogger.info("Working Directory: {}", workingDir.getAbsolutePath());
+ cmdLogger.info("Command: {}", cmdBuilder.toString());
+
+ String gracefulShutdown = props.get(GRACEFUL_SHUTDOWN_PROP);
+ if (gracefulShutdown == null) {
+ gracefulShutdown = DEFAULT_GRACEFUL_SHUTDOWN_VALUE;
+ }
+
+ final int gracefulShutdownSeconds;
+ try {
+ gracefulShutdownSeconds = Integer.parseInt(gracefulShutdown);
+ } catch (final NumberFormatException nfe) {
+ throw new NumberFormatException("The '" + GRACEFUL_SHUTDOWN_PROP + "' property in Bootstrap Config File "
+ + bootstrapConfigAbsoluteFile.getAbsolutePath() + " has an invalid value. Must be a non-negative integer");
+ }
+
+ if (gracefulShutdownSeconds < 0) {
+ throw new NumberFormatException("The '" + GRACEFUL_SHUTDOWN_PROP + "' property in Bootstrap Config File "
+ + bootstrapConfigAbsoluteFile.getAbsolutePath() + " has an invalid value. Must be a non-negative integer");
+ }
+
+ Process process = builder.start();
+ handleLogging(process);
+ Long pid = getPid(process, cmdLogger);
+ if (pid != null) {
+ nifiPid = pid;
+ final Properties nifiProps = new Properties();
+ nifiProps.setProperty("pid", String.valueOf(nifiPid));
+ saveProperties(nifiProps, cmdLogger);
+ }
+
+ shutdownHook = new ShutdownHook(process, this, secretKey, gracefulShutdownSeconds, loggingExecutor);
+ final Runtime runtime = Runtime.getRuntime();
+ runtime.addShutdownHook(shutdownHook);
+
+ while (true) {
+ final boolean alive = isAlive(process);
+
+ if (alive) {
+ try {
+ Thread.sleep(1000L);
+ } catch (final InterruptedException ie) {
+ }
+ } else {
+ try {
+ runtime.removeShutdownHook(shutdownHook);
+ } catch (final IllegalStateException ise) {
+ // happens when already shutting down
+ }
+
+ if (autoRestartNiFi) {
+ final File statusFile = getStatusFile(defaultLogger);
+ if (!statusFile.exists()) {
+ defaultLogger.debug("Status File no longer exists. Will not restart NiFi");
+ return;
+ }
+
+ defaultLogger.warn("Apache NiFi appears to have died. Restarting...");
+ process = builder.start();
+ handleLogging(process);
+
+ pid = getPid(process, defaultLogger);
+ if (pid != null) {
+ nifiPid = pid;
+ final Properties nifiProps = new Properties();
+ nifiProps.setProperty("pid", String.valueOf(nifiPid));
+ saveProperties(nifiProps, defaultLogger);
+ }
+
+ shutdownHook = new ShutdownHook(process, this, secretKey, gracefulShutdownSeconds, loggingExecutor);
+ runtime.addShutdownHook(shutdownHook);
+
+ final boolean started = waitForStart();
+
+ if (started) {
+ defaultLogger.info("Successfully started Apache NiFi{}", (pid == null ? "" : " with PID " + pid));
+ } else {
+ defaultLogger.error("Apache NiFi does not appear to have started");
+ }
+ } else {
+ return;
+ }
+ }
+ }
+ }
+
+ private void handleLogging(final Process process) {
+ final Set<Future<?>> existingFutures = loggingFutures;
+ if (existingFutures != null) {
+ for (final Future<?> future : existingFutures) {
+ future.cancel(false);
+ }
+ }
+
+ final Future<?> stdOutFuture = loggingExecutor.submit(new Runnable() {
+ @Override
+ public void run() {
+ final Logger stdOutLogger = LoggerFactory.getLogger("org.apache.nifi.StdOut");
+ final InputStream in = process.getInputStream();
+ try (final BufferedReader reader = new BufferedReader(new InputStreamReader(in))) {
+ String line;
+ while ((line = reader.readLine()) != null) {
+ stdOutLogger.info(line);
+ }
+ } catch (IOException e) {
+ defaultLogger.error("Failed to read from NiFi's Standard Out stream", e);
+ }
+ }
+ });
+
+ final Future<?> stdErrFuture = loggingExecutor.submit(new Runnable() {
+ @Override
+ public void run() {
+ final Logger stdErrLogger = LoggerFactory.getLogger("org.apache.nifi.StdErr");
+ final InputStream in = process.getErrorStream();
+ try (final BufferedReader reader = new BufferedReader(new InputStreamReader(in))) {
+ String line;
+ while ((line = reader.readLine()) != null) {
+ stdErrLogger.error(line);
+ }
+ } catch (IOException e) {
+ defaultLogger.error("Failed to read from NiFi's Standard Error stream", e);
+ }
+ }
+ });
+
+ final Set<Future<?>> futures = new HashSet<>();
+ futures.add(stdOutFuture);
+ futures.add(stdErrFuture);
+ this.loggingFutures = futures;
+ }
+
+ private Long getPid(final Process process, final Logger logger) {
+ try {
+ final Class<?> procClass = process.getClass();
+ final Field pidField = procClass.getDeclaredField("pid");
+ pidField.setAccessible(true);
+ final Object pidObject = pidField.get(process);
+
+ logger.debug("PID Object = {}", pidObject);
+
+ if (pidObject instanceof Number) {
+ return ((Number) pidObject).longValue();
+ }
+ return null;
+ } catch (final IllegalAccessException | NoSuchFieldException nsfe) {
+ logger.debug("Could not find PID for child process due to {}", nsfe);
+ return null;
+ }
+ }
+
+ private boolean isWindows() {
+ final String osName = System.getProperty("os.name");
+ return osName != null && osName.toLowerCase().contains("win");
+ }
+
+ private boolean waitForStart() {
+ lock.lock();
+ try {
+ final long startTime = System.nanoTime();
+
+ while (ccPort < 1) {
+ try {
+ startupCondition.await(1, TimeUnit.SECONDS);
+ } catch (final InterruptedException ie) {
+ return false;
+ }
+
+ final long waitNanos = System.nanoTime() - startTime;
+ final long waitSeconds = TimeUnit.NANOSECONDS.toSeconds(waitNanos);
+ if (waitSeconds > STARTUP_WAIT_SECONDS) {
+ return false;
+ }
+ }
+ } finally {
+ lock.unlock();
+ }
+ return true;
+ }
+
+ private File getFile(final String filename, final File workingDir) {
+ File file = new File(filename);
+ if (!file.isAbsolute()) {
+ file = new File(workingDir, filename);
+ }
+
+ return file;
+ }
+
+ private String replaceNull(final String value, final String replacement) {
+ return (value == null) ? replacement : value;
+ }
+
+ void setAutoRestartNiFi(final boolean restart) {
+ this.autoRestartNiFi = restart;
+ }
+
+ void setNiFiCommandControlPort(final int port, final String secretKey) {
+ this.ccPort = port;
+ this.secretKey = secretKey;
+
+ if (shutdownHook != null) {
+ shutdownHook.setSecretKey(secretKey);
+ }
+
+ final File statusFile = getStatusFile(defaultLogger);
+
+ final Properties nifiProps = new Properties();
+ if (nifiPid != -1) {
+ nifiProps.setProperty("pid", String.valueOf(nifiPid));
+ }
+ nifiProps.setProperty("port", String.valueOf(ccPort));
+ nifiProps.setProperty("secret.key", secretKey);
+
+ try {
+ saveProperties(nifiProps, defaultLogger);
+ } catch (final IOException ioe) {
+ defaultLogger.warn("Apache NiFi has started but failed to persist NiFi Port information to {} due to {}", new Object[]{statusFile.getAbsolutePath(), ioe});
+ }
+
+ defaultLogger.info("Apache NiFi now running and listening for Bootstrap requests on port {}", port);
+ }
+
+ int getNiFiCommandControlPort() {
+ return this.ccPort;
+ }
+
+ private static class Status {
+
+ private final Integer port;
+ private final String pid;
+
+ private final Boolean respondingToPing;
+ private final Boolean processRunning;
+
+ public Status(final Integer port, final String pid, final Boolean respondingToPing, final Boolean processRunning) {
+ this.port = port;
+ this.pid = pid;
+ this.respondingToPing = respondingToPing;
+ this.processRunning = processRunning;
+ }
+
+ public String getPid() {
+ return pid;
+ }
+
+ public Integer getPort() {
+ return port;
+ }
+
+ public boolean isRespondingToPing() {
+ return Boolean.TRUE.equals(respondingToPing);
+ }
+
+ public boolean isProcessRunning() {
+ return Boolean.TRUE.equals(processRunning);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/ShutdownHook.java
----------------------------------------------------------------------
diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/ShutdownHook.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/ShutdownHook.java
new file mode 100644
index 0000000..a594f60
--- /dev/null
+++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/ShutdownHook.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.bootstrap;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.Socket;
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+
+public class ShutdownHook extends Thread {
+
+ private final Process nifiProcess;
+ private final RunNiFi runner;
+ private final int gracefulShutdownSeconds;
+ private final ExecutorService executor;
+
+ private volatile String secretKey;
+
+ public ShutdownHook(final Process nifiProcess, final RunNiFi runner, final String secretKey, final int gracefulShutdownSeconds, final ExecutorService executor) {
+ this.nifiProcess = nifiProcess;
+ this.runner = runner;
+ this.secretKey = secretKey;
+ this.gracefulShutdownSeconds = gracefulShutdownSeconds;
+ this.executor = executor;
+ }
+
+ void setSecretKey(final String secretKey) {
+ this.secretKey = secretKey;
+ }
+
+ @Override
+ public void run() {
+ executor.shutdown();
+ runner.setAutoRestartNiFi(false);
+ final int ccPort = runner.getNiFiCommandControlPort();
+ if (ccPort > 0) {
+ System.out.println("Initiating Shutdown of NiFi...");
+
+ try {
+ final Socket socket = new Socket("localhost", ccPort);
+ final OutputStream out = socket.getOutputStream();
+ out.write(("SHUTDOWN " + secretKey + "\n").getBytes(StandardCharsets.UTF_8));
+ out.flush();
+
+ socket.close();
+ } catch (final IOException ioe) {
+ System.out.println("Failed to Shutdown NiFi due to " + ioe);
+ }
+ }
+
+ System.out.println("Waiting for Apache NiFi to finish shutting down...");
+ final long startWait = System.nanoTime();
+ while (RunNiFi.isAlive(nifiProcess)) {
+ final long waitNanos = System.nanoTime() - startWait;
+ final long waitSeconds = TimeUnit.NANOSECONDS.toSeconds(waitNanos);
+ if (waitSeconds >= gracefulShutdownSeconds && gracefulShutdownSeconds > 0) {
+ if (RunNiFi.isAlive(nifiProcess)) {
+ System.out.println("NiFi has not finished shutting down after " + gracefulShutdownSeconds + " seconds. Killing process.");
+ nifiProcess.destroy();
+ }
+ break;
+ } else {
+ try {
+ Thread.sleep(1000L);
+ } catch (final InterruptedException ie) {
+ }
+ }
+ }
+
+ final File statusFile = runner.getStatusFile();
+ if (!statusFile.delete()) {
+ System.err.println("Failed to delete status file " + statusFile.getAbsolutePath() + "; this file should be cleaned up manually");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/exception/InvalidCommandException.java
----------------------------------------------------------------------
diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/exception/InvalidCommandException.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/exception/InvalidCommandException.java
new file mode 100644
index 0000000..52e36b9
--- /dev/null
+++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/exception/InvalidCommandException.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.bootstrap.exception;
+
+public class InvalidCommandException extends Exception {
+
+ private static final long serialVersionUID = 1L;
+
+ public InvalidCommandException() {
+ super();
+ }
+
+ public InvalidCommandException(final String message) {
+ super(message);
+ }
+
+ public InvalidCommandException(final Throwable t) {
+ super(t);
+ }
+
+ public InvalidCommandException(final String message, final Throwable t) {
+ super(message, t);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/util/LimitingInputStream.java
----------------------------------------------------------------------
diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/util/LimitingInputStream.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/util/LimitingInputStream.java
new file mode 100644
index 0000000..2149342
--- /dev/null
+++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/util/LimitingInputStream.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.bootstrap.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+public class LimitingInputStream extends InputStream {
+
+ private final InputStream in;
+ private final long limit;
+ private long bytesRead = 0;
+
+ public LimitingInputStream(final InputStream in, final long limit) {
+ this.in = in;
+ this.limit = limit;
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (bytesRead >= limit) {
+ return -1;
+ }
+
+ final int val = in.read();
+ if (val > -1) {
+ bytesRead++;
+ }
+ return val;
+ }
+
+ @Override
+ public int read(final byte[] b) throws IOException {
+ if (bytesRead >= limit) {
+ return -1;
+ }
+
+ final int maxToRead = (int) Math.min(b.length, limit - bytesRead);
+
+ final int val = in.read(b, 0, maxToRead);
+ if (val > 0) {
+ bytesRead += val;
+ }
+ return val;
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ if (bytesRead >= limit) {
+ return -1;
+ }
+
+ final int maxToRead = (int) Math.min(len, limit - bytesRead);
+
+ final int val = in.read(b, off, maxToRead);
+ if (val > 0) {
+ bytesRead += val;
+ }
+ return val;
+ }
+
+ @Override
+ public long skip(final long n) throws IOException {
+ final long skipped = in.skip(Math.min(n, limit - bytesRead));
+ bytesRead += skipped;
+ return skipped;
+ }
+
+ @Override
+ public int available() throws IOException {
+ return in.available();
+ }
+
+ @Override
+ public void close() throws IOException {
+ in.close();
+ }
+
+ @Override
+ public void mark(int readlimit) {
+ in.mark(readlimit);
+ }
+
+ @Override
+ public boolean markSupported() {
+ return in.markSupported();
+ }
+
+ @Override
+ public void reset() throws IOException {
+ in.reset();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-data-provenance-utils/.gitignore
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-data-provenance-utils/.gitignore b/nifi-commons/nifi-data-provenance-utils/.gitignore
new file mode 100755
index 0000000..19f2e00
--- /dev/null
+++ b/nifi-commons/nifi-data-provenance-utils/.gitignore
@@ -0,0 +1,2 @@
+/target
+/target
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-data-provenance-utils/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-data-provenance-utils/pom.xml b/nifi-commons/nifi-data-provenance-utils/pom.xml
new file mode 100644
index 0000000..5481cd2
--- /dev/null
+++ b/nifi-commons/nifi-data-provenance-utils/pom.xml
@@ -0,0 +1,34 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-commons</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>nifi-data-provenance-utils</artifactId>
+ <packaging>jar</packaging>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-utils</artifactId>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncLineageSubmission.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncLineageSubmission.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncLineageSubmission.java
new file mode 100644
index 0000000..4a52a89
--- /dev/null
+++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncLineageSubmission.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import java.util.Collection;
+import java.util.Date;
+import java.util.UUID;
+
+import org.apache.nifi.provenance.lineage.ComputeLineageSubmission;
+import org.apache.nifi.provenance.lineage.LineageComputationType;
+
+/**
+ *
+ */
+public class AsyncLineageSubmission implements ComputeLineageSubmission {
+
+ private final String lineageIdentifier = UUID.randomUUID().toString();
+ private final Date submissionTime = new Date();
+
+ private final LineageComputationType computationType;
+ private final Long eventId;
+ private final Collection<String> lineageFlowFileUuids;
+
+ private volatile boolean canceled = false;
+
+ private final StandardLineageResult result;
+
+ public AsyncLineageSubmission(final LineageComputationType computationType, final Long eventId, final Collection<String> lineageFlowFileUuids, final int numSteps) {
+ this.computationType = computationType;
+ this.eventId = eventId;
+ this.lineageFlowFileUuids = lineageFlowFileUuids;
+ this.result = new StandardLineageResult(numSteps, lineageFlowFileUuids);
+ }
+
+ @Override
+ public StandardLineageResult getResult() {
+ return result;
+ }
+
+ @Override
+ public Date getSubmissionTime() {
+ return submissionTime;
+ }
+
+ @Override
+ public String getLineageIdentifier() {
+ return lineageIdentifier;
+ }
+
+ @Override
+ public void cancel() {
+ this.canceled = true;
+ }
+
+ @Override
+ public boolean isCanceled() {
+ return canceled;
+ }
+
+ @Override
+ public LineageComputationType getLineageComputationType() {
+ return computationType;
+ }
+
+ @Override
+ public Long getExpandedEventId() {
+ return eventId;
+ }
+
+ @Override
+ public Collection<String> getLineageFlowFileUuids() {
+ return lineageFlowFileUuids;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncQuerySubmission.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncQuerySubmission.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncQuerySubmission.java
new file mode 100644
index 0000000..00c6170
--- /dev/null
+++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncQuerySubmission.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import java.util.Date;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.nifi.provenance.search.Query;
+import org.apache.nifi.provenance.search.QuerySubmission;
+
+public class AsyncQuerySubmission implements QuerySubmission {
+
+ public static final int TTL = (int) TimeUnit.MILLISECONDS.convert(60, TimeUnit.SECONDS);
+
+ private final Date submissionTime = new Date();
+ private final Query query;
+
+ private volatile boolean canceled = false;
+ private final StandardQueryResult queryResult;
+
+ /**
+ * Constructs an AsyncQuerySubmission with the given query and the given
+ * number of steps, indicating how many results must be added to this
+ * AsyncQuerySubmission before it is considered finished
+ *
+ * @param query the query to execute
+ * @param numSteps how many steps to include
+ */
+ public AsyncQuerySubmission(final Query query, final int numSteps) {
+ this.query = query;
+ queryResult = new StandardQueryResult(query, numSteps);
+ }
+
+ @Override
+ public Date getSubmissionTime() {
+ return submissionTime;
+ }
+
+ @Override
+ public String getQueryIdentifier() {
+ return query.getIdentifier();
+ }
+
+ @Override
+ public void cancel() {
+ this.canceled = true;
+ queryResult.cancel();
+ }
+
+ @Override
+ public boolean isCanceled() {
+ return canceled;
+ }
+
+ @Override
+ public Query getQuery() {
+ return query;
+ }
+
+ @Override
+ public StandardQueryResult getResult() {
+ return queryResult;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/NamedSearchableField.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/NamedSearchableField.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/NamedSearchableField.java
new file mode 100644
index 0000000..dc2903f
--- /dev/null
+++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/NamedSearchableField.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import org.apache.nifi.provenance.search.SearchableField;
+import org.apache.nifi.provenance.search.SearchableFieldType;
+
+import static java.util.Objects.requireNonNull;
+
+/**
+ *
+ */
+public class NamedSearchableField implements SearchableField {
+
+ private final String identifier;
+ private final String searchableName;
+ private final SearchableFieldType fieldType;
+ private final String friendlyName;
+ private final boolean attribute;
+
+ NamedSearchableField(final String identifier, final String searchableName, final String friendlyName, final boolean attribute) {
+ this(identifier, searchableName, friendlyName, attribute, SearchableFieldType.STRING);
+ }
+
+ NamedSearchableField(final String identifier, final String searchableName, final String friendlyName, final boolean attribute, final SearchableFieldType fieldType) {
+ this.identifier = requireNonNull(identifier);
+ this.searchableName = requireNonNull(searchableName);
+ this.friendlyName = requireNonNull(friendlyName);
+ this.attribute = requireNonNull(attribute);
+ this.fieldType = requireNonNull(fieldType);
+ }
+
+ @Override
+ public String getIdentifier() {
+ return identifier;
+ }
+
+ @Override
+ public String getSearchableFieldName() {
+ return searchableName;
+ }
+
+ @Override
+ public String getFriendlyName() {
+ return friendlyName;
+ }
+
+ @Override
+ public boolean isAttribute() {
+ return attribute;
+ }
+
+ @Override
+ public SearchableFieldType getFieldType() {
+ return fieldType;
+ }
+
+ @Override
+ public String toString() {
+ return friendlyName;
+ }
+
+ @Override
+ public int hashCode() {
+ return 298347 + searchableName.hashCode() + (attribute ? 1 : 0);
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (obj == null) {
+ return false;
+ }
+
+ if (!(obj instanceof SearchableField)) {
+ return false;
+ }
+
+ final SearchableField other = (SearchableField) obj;
+ return (this.searchableName.equals(other.getSearchableFieldName()) && attribute == other.isAttribute());
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFieldParser.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFieldParser.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFieldParser.java
new file mode 100644
index 0000000..6a934b1
--- /dev/null
+++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFieldParser.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.nifi.provenance.search.SearchableField;
+
+public class SearchableFieldParser {
+
+ public static List<SearchableField> extractSearchableFields(final String indexedFieldString, final boolean predefinedField) {
+ final List<SearchableField> searchableFields = new ArrayList<>();
+ if (indexedFieldString != null) {
+ final String[] split = indexedFieldString.split(",");
+ for (String fieldName : split) {
+ fieldName = fieldName.trim();
+ if (fieldName.isEmpty()) {
+ continue;
+ }
+
+ final SearchableField searchableField;
+ if (predefinedField) {
+ searchableField = SearchableFields.getSearchableField(fieldName);
+ } else {
+ searchableField = SearchableFields.newSearchableAttribute(fieldName);
+ }
+
+ if (searchableField == null) {
+ throw new RuntimeException("Invalid Configuration: Provenance Repository configured to Index field '" + fieldName + "', but this is not a valid field");
+ }
+ searchableFields.add(searchableField);
+ }
+ }
+
+ return searchableFields;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFields.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFields.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFields.java
new file mode 100644
index 0000000..de62bca
--- /dev/null
+++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFields.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import org.apache.nifi.provenance.search.SearchableField;
+import org.apache.nifi.provenance.search.SearchableFieldType;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ *
+ */
+public class SearchableFields {
+
+ public static final SearchableField Identifier = new NamedSearchableField("Identifier", "identifier", "Identifier", false);
+ public static final SearchableField EventTime = new NamedSearchableField("EventTime", "time", "Event Time", false, SearchableFieldType.DATE);
+ public static final SearchableField FlowFileUUID = new NamedSearchableField("FlowFileUUID", "uuid", "FlowFile UUID", false);
+ public static final SearchableField Filename = new NamedSearchableField("Filename", "filename", "Filename", false);
+ public static final SearchableField EventType = new NamedSearchableField("EventType", "eventType", "Event Type", false);
+ public static final SearchableField TransitURI = new NamedSearchableField("TransitURI", "transitUri", "Transit URI", false);
+ public static final SearchableField ComponentID = new NamedSearchableField("ProcessorID", "processorId", "Component ID", false);
+ public static final SearchableField AlternateIdentifierURI = new NamedSearchableField("AlternateIdentifierURI", "alternateIdentifierUri", "Alternate Identifier URI", false);
+ public static final SearchableField FileSize = new NamedSearchableField("FileSize", "fileSize", "File Size", false, SearchableFieldType.DATA_SIZE);
+ public static final SearchableField Details = new NamedSearchableField("Details", "details", "Details", false, SearchableFieldType.STRING);
+ public static final SearchableField Relationship = new NamedSearchableField("Relationship", "relationship", "Relationship", false, SearchableFieldType.STRING);
+
+ public static final SearchableField LineageStartDate
+ = new NamedSearchableField("LineageStartDate", "lineageStartDate", "Lineage Start Date", false, SearchableFieldType.DATE);
+ public static final SearchableField LineageIdentifier
+ = new NamedSearchableField("LineageIdentifiers", "lineageIdentifier", "Lineage Identifier", false, SearchableFieldType.STRING);
+
+ public static final SearchableField ContentClaimSection
+ = new NamedSearchableField("ContentClaimSection", "contentClaimSection", "Content Claim Section", false, SearchableFieldType.STRING);
+ public static final SearchableField ContentClaimContainer
+ = new NamedSearchableField("ContentClaimContainer", "contentClaimContainer", "Content Claim Container", false, SearchableFieldType.STRING);
+ public static final SearchableField ContentClaimIdentifier
+ = new NamedSearchableField("ContentClaimIdentifier", "contentClaimIdentifier", "Content Claim Identifier", false, SearchableFieldType.STRING);
+ public static final SearchableField ContentClaimOffset
+ = new NamedSearchableField("ContentClaimOffset", "contentClaimOffset", "Content Claim Offset", false, SearchableFieldType.LONG);
+ public static final SearchableField SourceQueueIdentifier
+ = new NamedSearchableField("SourceQueueIdentifier", "sourceQueueIdentifier", "Source Queue Identifier", false, SearchableFieldType.STRING);
+
+ private static final Map<String, SearchableField> standardFields;
+
+ static {
+ final SearchableField[] searchableFields = new SearchableField[]{
+ EventTime, FlowFileUUID, Filename, EventType, TransitURI,
+ ComponentID, AlternateIdentifierURI, FileSize, Relationship, Details,
+ LineageStartDate, LineageIdentifier, ContentClaimSection, ContentClaimContainer, ContentClaimIdentifier,
+ ContentClaimOffset, SourceQueueIdentifier};
+
+ final Map<String, SearchableField> fields = new HashMap<>();
+ for (final SearchableField field : searchableFields) {
+ fields.put(field.getIdentifier(), field);
+ }
+
+ standardFields = Collections.unmodifiableMap(fields);
+ }
+
+ private SearchableFields() {
+ }
+
+ public static Collection<SearchableField> getStandardFields() {
+ return standardFields.values();
+ }
+
+ public static SearchableField getSearchableField(final String fieldIdentifier) {
+ return standardFields.get(fieldIdentifier);
+ }
+
+ public static SearchableField newSearchableAttribute(final String attributeName) {
+ return new NamedSearchableField(attributeName, attributeName, attributeName, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardLineageResult.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardLineageResult.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardLineageResult.java
new file mode 100644
index 0000000..63c53d0
--- /dev/null
+++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardLineageResult.java
@@ -0,0 +1,326 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.nifi.provenance.lineage.ComputeLineageResult;
+import org.apache.nifi.provenance.lineage.EdgeNode;
+import org.apache.nifi.provenance.lineage.EventNode;
+import org.apache.nifi.provenance.lineage.FlowFileNode;
+import org.apache.nifi.provenance.lineage.LineageEdge;
+import org.apache.nifi.provenance.lineage.LineageNode;
+
+/**
+ *
+ */
+public class StandardLineageResult implements ComputeLineageResult {
+
+ public static final int TTL = (int) TimeUnit.MILLISECONDS.convert(30, TimeUnit.MINUTES);
+ private static final Logger logger = LoggerFactory.getLogger(StandardLineageResult.class);
+
+ private final Collection<String> flowFileUuids;
+ private final Collection<ProvenanceEventRecord> relevantRecords = new ArrayList<>();
+ private final Set<LineageNode> nodes = new HashSet<>();
+ private final Set<LineageEdge> edges = new HashSet<>();
+ private final int numSteps;
+ private final long creationNanos;
+ private long computationNanos;
+
+ private final ReadWriteLock rwLock = new ReentrantReadWriteLock();
+ private final Lock readLock = rwLock.readLock();
+ private final Lock writeLock = rwLock.writeLock();
+
+ private Date expirationDate = null;
+ private String error = null;
+ private int numCompletedSteps = 0;
+
+ private volatile boolean canceled = false;
+
+ public StandardLineageResult(final int numSteps, final Collection<String> flowFileUuids) {
+ this.numSteps = numSteps;
+ this.creationNanos = System.nanoTime();
+ this.flowFileUuids = flowFileUuids;
+
+ updateExpiration();
+ }
+
+ @Override
+ public List<LineageNode> getNodes() {
+ readLock.lock();
+ try {
+ return new ArrayList<>(nodes);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public List<LineageEdge> getEdges() {
+ readLock.lock();
+ try {
+ return new ArrayList<>(edges);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ public int getNumberOfEdges() {
+ readLock.lock();
+ try {
+ return edges.size();
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ public int getNumberOfNodes() {
+ readLock.lock();
+ try {
+ return nodes.size();
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ public long getComputationTime(final TimeUnit timeUnit) {
+ readLock.lock();
+ try {
+ return timeUnit.convert(computationNanos, TimeUnit.NANOSECONDS);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public Date getExpiration() {
+ readLock.lock();
+ try {
+ return expirationDate;
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public String getError() {
+ readLock.lock();
+ try {
+ return error;
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public int getPercentComplete() {
+ readLock.lock();
+ try {
+ return (numSteps < 1) ? 100 : (int) (((float) numCompletedSteps / (float) numSteps) * 100.0F);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public boolean isFinished() {
+ readLock.lock();
+ try {
+ return numCompletedSteps >= numSteps || canceled;
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ public void setError(final String error) {
+ writeLock.lock();
+ try {
+ this.error = error;
+ numCompletedSteps++;
+
+ updateExpiration();
+
+ if (numCompletedSteps >= numSteps) {
+ computationNanos = System.nanoTime() - creationNanos;
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ public void update(final Collection<ProvenanceEventRecord> records) {
+ writeLock.lock();
+ try {
+ relevantRecords.addAll(records);
+
+ numCompletedSteps++;
+ updateExpiration();
+
+ if (numCompletedSteps >= numSteps && error == null) {
+ computeLineage();
+ computationNanos = System.nanoTime() - creationNanos;
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ /**
+ * Computes the lineage from the relevant Provenance Event Records. This
+ * method must be called with the write lock held and is only going to be
+ * useful after all of the records have been successfully obtained
+ */
+ private void computeLineage() {
+ final long startNanos = System.nanoTime();
+
+ nodes.clear();
+ edges.clear();
+
+ Map<String, LineageNode> lastEventMap = new HashMap<>(); // maps FlowFile UUID to last event for that FlowFile
+ final List<ProvenanceEventRecord> sortedRecords = new ArrayList<>(relevantRecords);
+ Collections.sort(sortedRecords, new Comparator<ProvenanceEventRecord>() {
+ @Override
+ public int compare(final ProvenanceEventRecord o1, final ProvenanceEventRecord o2) {
+ // Sort on Event Time, then Event ID.
+ final int eventTimeComparison = Long.compare(o1.getEventTime(), o2.getEventTime());
+ if (eventTimeComparison == 0) {
+ return Long.compare(o1.getEventId(), o2.getEventId());
+ } else {
+ return eventTimeComparison;
+ }
+ }
+ });
+
+ // convert the StandardProvenanceRecord objects into Lineage nodes (FlowFileNode, EventNodes).
+ for (final ProvenanceEventRecord record : sortedRecords) {
+ final LineageNode lineageNode = new EventNode(record);
+ final boolean added = nodes.add(lineageNode);
+ if (!added) {
+ logger.debug("Did not add {} because it already exists in the 'nodes' set", lineageNode);
+ }
+
+ // Create an edge that connects this node to the previous node for the same FlowFile UUID.
+ final LineageNode lastNode = lastEventMap.get(record.getFlowFileUuid());
+ if (lastNode != null) {
+ // We calculate the Edge UUID based on whether or not this event is a SPAWN.
+ // If this event is a SPAWN, then we want to use the previous node's UUID because a
+ // SPAWN Event's UUID is not necessarily what we want, since a SPAWN Event's UUID pertains to
+ // only one of (potentially) many UUIDs associated with the event. Otherwise, we know that
+ // the UUID of this record is appropriate, so we just use it.
+ final String edgeUuid;
+
+ switch (record.getEventType()) {
+ case JOIN:
+ case CLONE:
+ case REPLAY:
+ edgeUuid = lastNode.getFlowFileUuid();
+ break;
+ default:
+ edgeUuid = record.getFlowFileUuid();
+ break;
+ }
+
+ edges.add(new EdgeNode(edgeUuid, lastNode, lineageNode));
+ }
+
+ lastEventMap.put(record.getFlowFileUuid(), lineageNode);
+
+ switch (record.getEventType()) {
+ case FORK:
+ case JOIN:
+ case REPLAY:
+ case CLONE: {
+ // For events that create FlowFile nodes, we need to create the FlowFile Nodes and associated Edges, as appropriate
+ for (final String childUuid : record.getChildUuids()) {
+ if (flowFileUuids.contains(childUuid)) {
+ final FlowFileNode childNode = new FlowFileNode(childUuid, record.getEventTime());
+ final boolean isNewFlowFile = nodes.add(childNode);
+ if (!isNewFlowFile) {
+ final String msg = "Unable to generate Lineage Graph because multiple "
+ + "events were registered claiming to have generated the same FlowFile (UUID = " + childNode.getFlowFileUuid() + ")";
+ logger.error(msg);
+ setError(msg);
+ return;
+ }
+
+ edges.add(new EdgeNode(childNode.getFlowFileUuid(), lineageNode, childNode));
+ lastEventMap.put(childUuid, childNode);
+ }
+ }
+ for (final String parentUuid : record.getParentUuids()) {
+ LineageNode lastNodeForParent = lastEventMap.get(parentUuid);
+ if (lastNodeForParent != null && !lastNodeForParent.equals(lineageNode)) {
+ edges.add(new EdgeNode(parentUuid, lastNodeForParent, lineageNode));
+ }
+
+ lastEventMap.put(parentUuid, lineageNode);
+ }
+ }
+ break;
+ case RECEIVE:
+ case CREATE: {
+ // for a receive event, we want to create a FlowFile Node that represents the FlowFile received
+ // and create an edge from the Receive Event to the FlowFile Node
+ final LineageNode flowFileNode = new FlowFileNode(record.getFlowFileUuid(), record.getEventTime());
+ final boolean isNewFlowFile = nodes.add(flowFileNode);
+ if (!isNewFlowFile) {
+ final String msg = "Found cycle in graph. This indicates that multiple events "
+ + "were registered claiming to have generated the same FlowFile (UUID = " + flowFileNode.getFlowFileUuid() + ")";
+ setError(msg);
+ logger.error(msg);
+ return;
+ }
+ edges.add(new EdgeNode(record.getFlowFileUuid(), lineageNode, flowFileNode));
+ lastEventMap.put(record.getFlowFileUuid(), flowFileNode);
+ }
+ break;
+ default:
+ break;
+ }
+ }
+
+ final long nanos = System.nanoTime() - startNanos;
+ logger.debug("Finished building lineage with {} nodes and {} edges in {} millis", nodes.size(), edges.size(), TimeUnit.NANOSECONDS.toMillis(nanos));
+ }
+
+ void cancel() {
+ this.canceled = true;
+ }
+
+ /**
+ * Must be called with write lock!
+ */
+ private void updateExpiration() {
+ expirationDate = new Date(System.currentTimeMillis() + TTL);
+ }
+}
[36/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/LessThanEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/LessThanEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/LessThanEvaluator.java
new file mode 100644
index 0000000..e9bf231
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/LessThanEvaluator.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+
+public class LessThanEvaluator extends BooleanEvaluator {
+
+ private final Evaluator<Long> subject;
+ private final Evaluator<Long> comparison;
+
+ public LessThanEvaluator(final Evaluator<Long> subject, final Evaluator<Long> comparison) {
+ this.subject = subject;
+ this.comparison = comparison;
+ }
+
+ @Override
+ public QueryResult<Boolean> evaluate(final Map<String, String> attributes) {
+ final Long subjectValue = subject.evaluate(attributes).getValue();
+ if (subjectValue == null) {
+ return new BooleanQueryResult(false);
+ }
+
+ final Long comparisonValue = comparison.evaluate(attributes).getValue();
+ if (comparisonValue == null) {
+ return new BooleanQueryResult(false);
+ }
+
+ return new BooleanQueryResult(subjectValue < comparisonValue);
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/LessThanOrEqualEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/LessThanOrEqualEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/LessThanOrEqualEvaluator.java
new file mode 100644
index 0000000..5bf5ee5
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/LessThanOrEqualEvaluator.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+
+public class LessThanOrEqualEvaluator extends BooleanEvaluator {
+
+ private final Evaluator<Long> subject;
+ private final Evaluator<Long> comparison;
+
+ public LessThanOrEqualEvaluator(final Evaluator<Long> subject, final Evaluator<Long> comparison) {
+ this.subject = subject;
+ this.comparison = comparison;
+ }
+
+ @Override
+ public QueryResult<Boolean> evaluate(final Map<String, String> attributes) {
+ final Long subjectValue = subject.evaluate(attributes).getValue();
+ if (subjectValue == null) {
+ return new BooleanQueryResult(false);
+ }
+
+ final Long comparisonValue = comparison.evaluate(attributes).getValue();
+ if (comparisonValue == null) {
+ return new BooleanQueryResult(false);
+ }
+
+ return new BooleanQueryResult(subjectValue <= comparisonValue);
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/MatchesEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/MatchesEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/MatchesEvaluator.java
new file mode 100644
index 0000000..6f017ea
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/MatchesEvaluator.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+import java.util.regex.Pattern;
+
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.literals.StringLiteralEvaluator;
+
+public class MatchesEvaluator extends BooleanEvaluator {
+
+ private final Evaluator<String> subject;
+ private final Evaluator<String> search;
+
+ private final Pattern compiledPattern;
+
+ public MatchesEvaluator(final Evaluator<String> subject, final Evaluator<String> search) {
+ this.subject = subject;
+ this.search = search;
+
+ // if the search string is a literal, we don't need to evaluate it each time; we can just
+ // pre-compile it. Otherwise, it must be compiled every time.
+ if (search instanceof StringLiteralEvaluator) {
+ this.compiledPattern = Pattern.compile(search.evaluate(null).getValue());
+ } else {
+ this.compiledPattern = null;
+ }
+ }
+
+ @Override
+ public QueryResult<Boolean> evaluate(final Map<String, String> attributes) {
+ final String subjectValue = subject.evaluate(attributes).getValue();
+ if (subjectValue == null) {
+ return new BooleanQueryResult(false);
+ }
+ final Pattern pattern;
+ if (compiledPattern == null) {
+ pattern = Pattern.compile(search.evaluate(attributes).getValue());
+ } else {
+ pattern = compiledPattern;
+ }
+
+ final boolean matches = pattern.matcher(subjectValue).matches();
+ return new BooleanQueryResult(matches);
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/MinusEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/MinusEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/MinusEvaluator.java
new file mode 100644
index 0000000..0555187
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/MinusEvaluator.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.NumberEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.NumberQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+
+public class MinusEvaluator extends NumberEvaluator {
+
+ private final Evaluator<Long> subject;
+ private final Evaluator<Long> minusValue;
+
+ public MinusEvaluator(final Evaluator<Long> subject, final Evaluator<Long> minusValue) {
+ this.subject = subject;
+ this.minusValue = minusValue;
+ }
+
+ @Override
+ public QueryResult<Long> evaluate(final Map<String, String> attributes) {
+ final Long subjectValue = subject.evaluate(attributes).getValue();
+ if (subjectValue == null) {
+ return new NumberQueryResult(null);
+ }
+
+ final Long minus = minusValue.evaluate(attributes).getValue();
+ if (minus == null) {
+ return new NumberQueryResult(null);
+ }
+
+ final long result = subjectValue - minus;
+ return new NumberQueryResult(result);
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ModEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ModEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ModEvaluator.java
new file mode 100644
index 0000000..cf2121c
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ModEvaluator.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.NumberEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.NumberQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+
+public class ModEvaluator extends NumberEvaluator {
+
+ private final Evaluator<Long> subject;
+ private final Evaluator<Long> modValue;
+
+ public ModEvaluator(final Evaluator<Long> subject, final Evaluator<Long> modValue) {
+ this.subject = subject;
+ this.modValue = modValue;
+ }
+
+ @Override
+ public QueryResult<Long> evaluate(final Map<String, String> attributes) {
+ final Long subjectValue = subject.evaluate(attributes).getValue();
+ if (subjectValue == null) {
+ return new NumberQueryResult(null);
+ }
+
+ final Long mod = modValue.evaluate(attributes).getValue();
+ if (mod == null) {
+ return new NumberQueryResult(null);
+ }
+
+ final long result = subjectValue % mod;
+ return new NumberQueryResult(result);
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/MultiplyEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/MultiplyEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/MultiplyEvaluator.java
new file mode 100644
index 0000000..4480c98
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/MultiplyEvaluator.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.NumberEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.NumberQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+
+public class MultiplyEvaluator extends NumberEvaluator {
+
+ private final Evaluator<Long> subject;
+ private final Evaluator<Long> multiplyValue;
+
+ public MultiplyEvaluator(final Evaluator<Long> subject, final Evaluator<Long> multiplyValue) {
+ this.subject = subject;
+ this.multiplyValue = multiplyValue;
+ }
+
+ @Override
+ public QueryResult<Long> evaluate(final Map<String, String> attributes) {
+ final Long subjectValue = subject.evaluate(attributes).getValue();
+ if (subjectValue == null) {
+ return new NumberQueryResult(null);
+ }
+
+ final Long multiply = multiplyValue.evaluate(attributes).getValue();
+ if (multiply == null) {
+ return new NumberQueryResult(null);
+ }
+
+ final long result = subjectValue * multiply;
+ return new NumberQueryResult(result);
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/NotEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/NotEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/NotEvaluator.java
new file mode 100644
index 0000000..2a09e33
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/NotEvaluator.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+
+public class NotEvaluator extends BooleanEvaluator {
+
+ private final Evaluator<Boolean> subjectEvaluator;
+
+ public NotEvaluator(final Evaluator<Boolean> subjectEvaluator) {
+ this.subjectEvaluator = subjectEvaluator;
+ }
+
+ @Override
+ public QueryResult<Boolean> evaluate(final Map<String, String> attributes) {
+ final QueryResult<Boolean> subjectValue = subjectEvaluator.evaluate(attributes);
+ if (subjectValue == null) {
+ return new BooleanQueryResult(null);
+ }
+
+ return new BooleanQueryResult(!subjectValue.getValue());
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subjectEvaluator;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/NotNullEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/NotNullEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/NotNullEvaluator.java
new file mode 100644
index 0000000..126fbfa
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/NotNullEvaluator.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+
+public class NotNullEvaluator extends BooleanEvaluator {
+
+ private final Evaluator<?> subject;
+
+ public NotNullEvaluator(final Evaluator<?> subject) {
+ this.subject = subject;
+ }
+
+ @Override
+ public QueryResult<Boolean> evaluate(final Map<String, String> attributes) {
+ final Object subjectValue = subject.evaluate(attributes).getValue();
+ return new BooleanQueryResult(subjectValue != null);
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/NowEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/NowEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/NowEvaluator.java
new file mode 100644
index 0000000..bfde6b0
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/NowEvaluator.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Date;
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.DateEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.DateQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+
+public class NowEvaluator extends DateEvaluator {
+
+ @Override
+ public QueryResult<Date> evaluate(final Map<String, String> attributes) {
+ return new DateQueryResult(new Date());
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/NumberToDateEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/NumberToDateEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/NumberToDateEvaluator.java
new file mode 100644
index 0000000..6867202
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/NumberToDateEvaluator.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Date;
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.DateEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.DateQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+
+public class NumberToDateEvaluator extends DateEvaluator {
+
+ private final Evaluator<Long> subject;
+
+ public NumberToDateEvaluator(final Evaluator<Long> subject) {
+ this.subject = subject;
+ }
+
+ @Override
+ public QueryResult<Date> evaluate(final Map<String, String> attributes) {
+ final QueryResult<Long> result = subject.evaluate(attributes);
+ final Long value = result.getValue();
+ if (value == null) {
+ return null;
+ }
+
+ return new DateQueryResult(new Date(value));
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/OneUpSequenceEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/OneUpSequenceEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/OneUpSequenceEvaluator.java
new file mode 100644
index 0000000..11d3fa9
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/OneUpSequenceEvaluator.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.NumberEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.NumberQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+
+public class OneUpSequenceEvaluator extends NumberEvaluator {
+
+ private static final AtomicLong value = new AtomicLong(0L);
+
+ @Override
+ public QueryResult<Long> evaluate(final Map<String, String> attributes) {
+ return new NumberQueryResult(value.getAndIncrement());
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/OrEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/OrEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/OrEvaluator.java
new file mode 100644
index 0000000..719fa11
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/OrEvaluator.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+
+public class OrEvaluator extends BooleanEvaluator {
+
+ private final Evaluator<Boolean> subjectEvaluator;
+ private final Evaluator<Boolean> rhsEvaluator;
+
+ public OrEvaluator(final Evaluator<Boolean> subjectEvaluator, final Evaluator<Boolean> rhsEvaluator) {
+ this.subjectEvaluator = subjectEvaluator;
+ this.rhsEvaluator = rhsEvaluator;
+ }
+
+ @Override
+ public QueryResult<Boolean> evaluate(final Map<String, String> attributes) {
+ final QueryResult<Boolean> subjectValue = subjectEvaluator.evaluate(attributes);
+ if (subjectValue == null) {
+ return new BooleanQueryResult(null);
+ }
+
+ if (Boolean.TRUE.equals(subjectValue.getValue())) {
+ return new BooleanQueryResult(true);
+ }
+
+ final QueryResult<Boolean> rhsValue = rhsEvaluator.evaluate(attributes);
+ if (rhsValue == null) {
+ return new BooleanQueryResult(false);
+ }
+
+ return new BooleanQueryResult(rhsValue.getValue());
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subjectEvaluator;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/PlusEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/PlusEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/PlusEvaluator.java
new file mode 100644
index 0000000..7c9278f
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/PlusEvaluator.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.NumberEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.NumberQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+
+public class PlusEvaluator extends NumberEvaluator {
+
+ private final Evaluator<Long> subject;
+ private final Evaluator<Long> plusValue;
+
+ public PlusEvaluator(final Evaluator<Long> subject, final Evaluator<Long> plusValue) {
+ this.subject = subject;
+ this.plusValue = plusValue;
+ }
+
+ @Override
+ public QueryResult<Long> evaluate(final Map<String, String> attributes) {
+ final Long subjectValue = subject.evaluate(attributes).getValue();
+ if (subjectValue == null) {
+ return new NumberQueryResult(null);
+ }
+
+ final Long plus = plusValue.evaluate(attributes).getValue();
+ if (plus == null) {
+ return new NumberQueryResult(null);
+ }
+
+ final long result = subjectValue + plus;
+ return new NumberQueryResult(result);
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/PrependEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/PrependEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/PrependEvaluator.java
new file mode 100644
index 0000000..1053328
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/PrependEvaluator.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult;
+
+public class PrependEvaluator extends StringEvaluator {
+
+ private final Evaluator<String> subject;
+ private final Evaluator<String> prependEvaluator;
+
+ public PrependEvaluator(final Evaluator<String> subject, final Evaluator<String> prepend) {
+ this.subject = subject;
+ this.prependEvaluator = prepend;
+ }
+
+ @Override
+ public QueryResult<String> evaluate(final Map<String, String> attributes) {
+ final String subjectValue = subject.evaluate(attributes).getValue();
+ final String prependValue = prependEvaluator.evaluate(attributes).getValue();
+
+ final String result = (prependValue == null ? "" : prependValue) + (subjectValue == null ? "" : subjectValue);
+ return new StringQueryResult(result);
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ReplaceAllEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ReplaceAllEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ReplaceAllEvaluator.java
new file mode 100644
index 0000000..b796233
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ReplaceAllEvaluator.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult;
+
+public class ReplaceAllEvaluator extends StringEvaluator {
+
+ private final Evaluator<String> subject;
+ private final Evaluator<String> search;
+ private final Evaluator<String> replacement;
+
+ public ReplaceAllEvaluator(final Evaluator<String> subject, final Evaluator<String> search, final Evaluator<String> replacement) {
+ this.subject = subject;
+ this.search = search;
+ this.replacement = replacement;
+ }
+
+ @Override
+ public QueryResult<String> evaluate(final Map<String, String> attributes) {
+ final String subjectValue = subject.evaluate(attributes).getValue();
+ if (subjectValue == null) {
+ return new StringQueryResult(null);
+ }
+ final String searchValue = search.evaluate(attributes).getValue();
+ final String replacementValue = replacement.evaluate(attributes).getValue();
+
+ return new StringQueryResult(subjectValue.replaceAll(searchValue, replacementValue));
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ReplaceEmptyEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ReplaceEmptyEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ReplaceEmptyEvaluator.java
new file mode 100644
index 0000000..9c2cb6d
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ReplaceEmptyEvaluator.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator;
+
+public class ReplaceEmptyEvaluator extends StringEvaluator {
+
+ private final Evaluator<String> subjectEvaluator;
+ private final Evaluator<String> replacementEvaluator;
+
+ public ReplaceEmptyEvaluator(final Evaluator<String> subjectEvaluator, final Evaluator<String> replacementEvaluator) {
+ this.subjectEvaluator = subjectEvaluator;
+ this.replacementEvaluator = replacementEvaluator;
+ }
+
+ @Override
+ public QueryResult<String> evaluate(final Map<String, String> attributes) {
+ final QueryResult<String> subjectResult = subjectEvaluator.evaluate(attributes);
+ final String subjectValue = subjectResult.getValue();
+ final boolean isEmpty = subjectValue == null || subjectValue.toString().trim().isEmpty();
+ if (isEmpty) {
+ return replacementEvaluator.evaluate(attributes);
+ } else {
+ return subjectResult;
+ }
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subjectEvaluator;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ReplaceEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ReplaceEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ReplaceEvaluator.java
new file mode 100644
index 0000000..e55e963
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ReplaceEvaluator.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult;
+
+public class ReplaceEvaluator extends StringEvaluator {
+
+ private final Evaluator<String> subject;
+ private final Evaluator<String> search;
+ private final Evaluator<String> replacement;
+
+ public ReplaceEvaluator(final Evaluator<String> subject, final Evaluator<String> search, final Evaluator<String> replacement) {
+ this.subject = subject;
+ this.search = search;
+ this.replacement = replacement;
+ }
+
+ @Override
+ public QueryResult<String> evaluate(final Map<String, String> attributes) {
+ final String subjectValue = subject.evaluate(attributes).getValue();
+ if (subjectValue == null) {
+ return new StringQueryResult(null);
+ }
+ final String searchValue = search.evaluate(attributes).getValue();
+ final String replacementValue = replacement.evaluate(attributes).getValue();
+
+ return new StringQueryResult(subjectValue.replace(searchValue, replacementValue));
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ReplaceNullEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ReplaceNullEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ReplaceNullEvaluator.java
new file mode 100644
index 0000000..e8487c6
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ReplaceNullEvaluator.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult;
+
+public class ReplaceNullEvaluator extends StringEvaluator {
+
+ private final Evaluator<String> subject;
+ private final Evaluator<String> resultEvaluator;
+
+ public ReplaceNullEvaluator(final Evaluator<String> subject, final Evaluator<String> resultEvaluator) {
+ this.subject = subject;
+ this.resultEvaluator = resultEvaluator;
+ }
+
+ @Override
+ public QueryResult<String> evaluate(final Map<String, String> attributes) {
+ final String subjectValue = subject.evaluate(attributes).getValue();
+ return new StringQueryResult(subjectValue == null ? resultEvaluator.evaluate(attributes).getValue() : subjectValue);
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/StartsWithEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/StartsWithEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/StartsWithEvaluator.java
new file mode 100644
index 0000000..6a43550
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/StartsWithEvaluator.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+
+public class StartsWithEvaluator extends BooleanEvaluator {
+
+ private final Evaluator<String> subject;
+ private final Evaluator<String> search;
+
+ public StartsWithEvaluator(final Evaluator<String> subject, final Evaluator<String> search) {
+ this.subject = subject;
+ this.search = search;
+ }
+
+ @Override
+ public QueryResult<Boolean> evaluate(final Map<String, String> attributes) {
+ final String subjectValue = subject.evaluate(attributes).getValue();
+ if (subjectValue == null) {
+ return new BooleanQueryResult(false);
+ }
+
+ final String searchString = search.evaluate(attributes).getValue();
+ return new BooleanQueryResult(searchString == null ? false : subjectValue.startsWith(searchString));
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/StringToDateEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/StringToDateEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/StringToDateEvaluator.java
new file mode 100644
index 0000000..590176e
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/StringToDateEvaluator.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Locale;
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.DateEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.DateQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.exception.IllegalAttributeException;
+
+public class StringToDateEvaluator extends DateEvaluator {
+
+ private final Evaluator<String> subject;
+ private final Evaluator<String> format;
+
+ public StringToDateEvaluator(final Evaluator<String> subject, final Evaluator<String> format) {
+ this.subject = subject;
+ this.format = format;
+ }
+
+ @Override
+ public QueryResult<Date> evaluate(final Map<String, String> attributes) {
+ final String subjectValue = subject.evaluate(attributes).getValue();
+ final String formatValue = format.evaluate(attributes).getValue();
+ if (subjectValue == null || formatValue == null) {
+ return new DateQueryResult(null);
+ }
+
+ try {
+ return new DateQueryResult(new SimpleDateFormat(formatValue, Locale.US).parse(subjectValue));
+ } catch (final ParseException e) {
+ throw new IllegalAttributeException("Cannot parse attribute value as a date; date format: "
+ + formatValue + "; attribute value: " + subjectValue);
+ } catch (final IllegalArgumentException e) {
+ throw new IllegalAttributeException("Invalid date format: " + formatValue);
+ }
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/SubstringAfterEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/SubstringAfterEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/SubstringAfterEvaluator.java
new file mode 100644
index 0000000..6625cb6
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/SubstringAfterEvaluator.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult;
+
+public class SubstringAfterEvaluator extends StringEvaluator {
+
+ private final Evaluator<String> subject;
+ private final Evaluator<String> afterEvaluator;
+
+ public SubstringAfterEvaluator(final Evaluator<String> subject, final Evaluator<String> afterEvaluator) {
+ this.subject = subject;
+ this.afterEvaluator = afterEvaluator;
+ }
+
+ @Override
+ public QueryResult<String> evaluate(final Map<String, String> attributes) {
+ final String subjectValue = subject.evaluate(attributes).getValue();
+ if (subjectValue == null) {
+ return new StringQueryResult("");
+ }
+ final String afterValue = afterEvaluator.evaluate(attributes).getValue();
+ if (afterValue == null || afterValue.length() == 0) {
+ return new StringQueryResult(subjectValue);
+ }
+
+ final int index = subjectValue.indexOf(afterValue);
+ if (index < 0) {
+ return new StringQueryResult(subjectValue);
+ }
+ return new StringQueryResult(subjectValue.substring(index + afterValue.length()));
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/SubstringAfterLastEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/SubstringAfterLastEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/SubstringAfterLastEvaluator.java
new file mode 100644
index 0000000..2cc9c3c
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/SubstringAfterLastEvaluator.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult;
+
+public class SubstringAfterLastEvaluator extends StringEvaluator {
+
+ private final Evaluator<String> subject;
+ private final Evaluator<String> afterEvaluator;
+
+ public SubstringAfterLastEvaluator(final Evaluator<String> subject, final Evaluator<String> afterEvaluator) {
+ this.subject = subject;
+ this.afterEvaluator = afterEvaluator;
+ }
+
+ @Override
+ public QueryResult<String> evaluate(final Map<String, String> attributes) {
+ final String subjectValue = subject.evaluate(attributes).getValue();
+ if (subjectValue == null) {
+ return new StringQueryResult("");
+ }
+ final String afterValue = afterEvaluator.evaluate(attributes).getValue();
+ final int index = subjectValue.lastIndexOf(afterValue);
+ if (index < 0 || index >= subjectValue.length()) {
+ return new StringQueryResult(subjectValue);
+ }
+ return new StringQueryResult(subjectValue.substring(index + afterValue.length()));
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/SubstringBeforeEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/SubstringBeforeEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/SubstringBeforeEvaluator.java
new file mode 100644
index 0000000..4b8c3d0
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/SubstringBeforeEvaluator.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult;
+
+public class SubstringBeforeEvaluator extends StringEvaluator {
+
+ private final Evaluator<String> subject;
+ private final Evaluator<String> beforeEvaluator;
+
+ public SubstringBeforeEvaluator(final Evaluator<String> subject, final Evaluator<String> beforeEvaluator) {
+ this.subject = subject;
+ this.beforeEvaluator = beforeEvaluator;
+ }
+
+ @Override
+ public QueryResult<String> evaluate(final Map<String, String> attributes) {
+ final String subjectValue = subject.evaluate(attributes).getValue();
+ if (subjectValue == null) {
+ return new StringQueryResult("");
+ }
+ final String beforeValue = beforeEvaluator.evaluate(attributes).getValue();
+ if (beforeValue == null || beforeValue.length() == 0) {
+ return new StringQueryResult(subjectValue);
+ }
+ final int index = subjectValue.indexOf(beforeValue);
+ if (index < 0) {
+ return new StringQueryResult(subjectValue);
+ }
+ return new StringQueryResult(subjectValue.substring(0, index));
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/SubstringBeforeLastEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/SubstringBeforeLastEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/SubstringBeforeLastEvaluator.java
new file mode 100644
index 0000000..14548c1
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/SubstringBeforeLastEvaluator.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult;
+
+public class SubstringBeforeLastEvaluator extends StringEvaluator {
+
+ private final Evaluator<String> subject;
+ private final Evaluator<String> beforeEvaluator;
+
+ public SubstringBeforeLastEvaluator(final Evaluator<String> subject, final Evaluator<String> beforeEvaluator) {
+ this.subject = subject;
+ this.beforeEvaluator = beforeEvaluator;
+ }
+
+ @Override
+ public QueryResult<String> evaluate(final Map<String, String> attributes) {
+ final String subjectValue = subject.evaluate(attributes).getValue();
+ if (subjectValue == null) {
+ return new StringQueryResult("");
+ }
+ final String beforeValue = beforeEvaluator.evaluate(attributes).getValue();
+ final int index = subjectValue.lastIndexOf(beforeValue);
+ if (index < 0) {
+ return new StringQueryResult(subjectValue);
+ }
+ return new StringQueryResult(subjectValue.substring(0, index));
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/SubstringEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/SubstringEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/SubstringEvaluator.java
new file mode 100644
index 0000000..71b78f1
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/SubstringEvaluator.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult;
+
+public class SubstringEvaluator extends StringEvaluator {
+
+ private final Evaluator<String> subject;
+ private final Evaluator<Long> startIndex;
+ private final Evaluator<Long> endIndex;
+
+ public SubstringEvaluator(final Evaluator<String> subject, final Evaluator<Long> startIndex, final Evaluator<Long> endIndex) {
+ this.subject = subject;
+ this.startIndex = startIndex;
+ this.endIndex = endIndex;
+ }
+
+ public SubstringEvaluator(final Evaluator<String> subject, final Evaluator<Long> startIndex) {
+ this.subject = subject;
+ this.startIndex = startIndex;
+ this.endIndex = null;
+ }
+
+ @Override
+ public QueryResult<String> evaluate(final Map<String, String> attributes) {
+ final String subjectValue = subject.evaluate(attributes).getValue();
+ if (subjectValue == null) {
+ return new StringQueryResult("");
+ }
+ final int startIndexValue = startIndex.evaluate(attributes).getValue().intValue();
+ if (endIndex == null) {
+ return new StringQueryResult(subjectValue.substring(startIndexValue));
+ } else {
+ final int endIndexValue = endIndex.evaluate(attributes).getValue().intValue();
+ return new StringQueryResult(subjectValue.substring(startIndexValue, endIndexValue));
+ }
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ToLowerEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ToLowerEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ToLowerEvaluator.java
new file mode 100644
index 0000000..33175f8
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ToLowerEvaluator.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult;
+
+public class ToLowerEvaluator extends StringEvaluator {
+
+ private final Evaluator<String> subject;
+
+ public ToLowerEvaluator(final Evaluator<String> subject) {
+ this.subject = subject;
+ }
+
+ @Override
+ public QueryResult<String> evaluate(final Map<String, String> attributes) {
+ final String subjectValue = subject.evaluate(attributes).getValue();
+ return new StringQueryResult(subjectValue == null ? null : subjectValue.toLowerCase());
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ToNumberEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ToNumberEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ToNumberEvaluator.java
new file mode 100644
index 0000000..736f0b9
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ToNumberEvaluator.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.NumberEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.NumberQueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+
+public class ToNumberEvaluator extends NumberEvaluator {
+
+ private final Evaluator<String> subject;
+
+ public ToNumberEvaluator(final Evaluator<String> subject) {
+ this.subject = subject;
+ }
+
+ @Override
+ public QueryResult<Long> evaluate(final Map<String, String> attributes) {
+ final String subjectValue = subject.evaluate(attributes).getValue();
+ return new NumberQueryResult(subjectValue == null || subjectValue.trim().isEmpty() ? null : Long.valueOf(subjectValue));
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ToRadixEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ToRadixEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ToRadixEvaluator.java
new file mode 100644
index 0000000..1cf7e55
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ToRadixEvaluator.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Arrays;
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult;
+
+public class ToRadixEvaluator extends StringEvaluator {
+
+ private final Evaluator<Long> numberEvaluator;
+ private final Evaluator<Long> radixEvaluator;
+ private final Evaluator<Long> minimumWidthEvaluator;
+
+ public ToRadixEvaluator(final Evaluator<Long> subject, final Evaluator<Long> radixEvaluator) {
+ this(subject, radixEvaluator, null);
+ }
+
+ public ToRadixEvaluator(final Evaluator<Long> subject, final Evaluator<Long> radixEvaluator, final Evaluator<Long> minimumWidthEvaluator) {
+ this.numberEvaluator = subject;
+ this.radixEvaluator = radixEvaluator;
+ this.minimumWidthEvaluator = minimumWidthEvaluator;
+ }
+
+ @Override
+ public QueryResult<String> evaluate(final Map<String, String> attributes) {
+ final Long result = numberEvaluator.evaluate(attributes).getValue();
+ if (result == null) {
+ return new StringQueryResult(null);
+ }
+
+ final Long radix = radixEvaluator.evaluate(attributes).getValue();
+ if (radix == null) {
+ return new StringQueryResult(null);
+ }
+
+ String stringValue = Long.toString(result.longValue(), radix.intValue());
+ if (minimumWidthEvaluator != null) {
+ final Long minimumWidth = minimumWidthEvaluator.evaluate(attributes).getValue();
+ if (minimumWidth != null) {
+ final int paddingWidth = minimumWidth.intValue() - stringValue.length();
+ if (paddingWidth > 0) {
+ final char[] padChars = new char[paddingWidth];
+ Arrays.fill(padChars, '0');
+ stringValue = String.valueOf(padChars) + stringValue;
+ }
+ }
+ }
+
+ return new StringQueryResult(stringValue);
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return numberEvaluator;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ToStringEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ToStringEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ToStringEvaluator.java
new file mode 100644
index 0000000..2f7fbe3
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ToStringEvaluator.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import java.util.Map;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult;
+
+public class ToStringEvaluator extends StringEvaluator {
+
+ private final Evaluator<?> subject;
+
+ public ToStringEvaluator(final Evaluator<?> subject) {
+ this.subject = subject;
+ }
+
+ @Override
+ public QueryResult<String> evaluate(final Map<String, String> attributes) {
+ final Object result = subject.evaluate(attributes).getValue();
+ return new StringQueryResult(result == null ? null : result.toString());
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return subject;
+ }
+
+}
[31/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-processor-utilities/src/main/java/org/apache/nifi/processor/util/FlowFileFilters.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-processor-utilities/src/main/java/org/apache/nifi/processor/util/FlowFileFilters.java b/nifi-commons/nifi-processor-utilities/src/main/java/org/apache/nifi/processor/util/FlowFileFilters.java
new file mode 100644
index 0000000..2d1a407
--- /dev/null
+++ b/nifi-commons/nifi-processor-utilities/src/main/java/org/apache/nifi/processor/util/FlowFileFilters.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor.util;
+
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.DataUnit;
+import org.apache.nifi.processor.FlowFileFilter;
+
+public class FlowFileFilters {
+
+ /**
+ * Returns a new {@link FlowFileFilter} that will pull FlowFiles until the
+ * maximum file size has been reached, or the maximum FlowFile Count was
+ * been reached (this is important because FlowFiles may be 0 bytes!). If
+ * the first FlowFile exceeds the max size, the FlowFile will be selected
+ * and no other FlowFile will be.
+ *
+ * @param maxSize the maximum size of the group of FlowFiles
+ * @param unit the unit of the <code>maxSize</code> argument
+ * @param maxCount the maximum number of FlowFiles to pull
+ * @return filter
+ */
+ public static FlowFileFilter newSizeBasedFilter(final double maxSize, final DataUnit unit, final int maxCount) {
+ final double maxBytes = DataUnit.B.convert(maxSize, unit);
+
+ return new FlowFileFilter() {
+ int count = 0;
+ long size = 0L;
+
+ @Override
+ public FlowFileFilterResult filter(final FlowFile flowFile) {
+ if (count == 0) {
+ count++;
+ size += flowFile.getSize();
+
+ return FlowFileFilterResult.ACCEPT_AND_CONTINUE;
+ }
+
+ if ((size + flowFile.getSize() > maxBytes) || (count + 1 > maxCount)) {
+ return FlowFileFilterResult.REJECT_AND_TERMINATE;
+ }
+
+ count++;
+ size += flowFile.getSize();
+ return FlowFileFilterResult.ACCEPT_AND_CONTINUE;
+ }
+
+ };
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-processor-utilities/src/main/java/org/apache/nifi/processor/util/SSLProperties.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-processor-utilities/src/main/java/org/apache/nifi/processor/util/SSLProperties.java b/nifi-commons/nifi-processor-utilities/src/main/java/org/apache/nifi/processor/util/SSLProperties.java
new file mode 100644
index 0000000..dca15b0
--- /dev/null
+++ b/nifi-commons/nifi-processor-utilities/src/main/java/org/apache/nifi/processor/util/SSLProperties.java
@@ -0,0 +1,236 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.security.KeyManagementException;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.UnrecoverableKeyException;
+import java.security.cert.CertificateException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import javax.net.ssl.SSLContext;
+
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.security.util.CertificateUtils;
+import org.apache.nifi.security.util.KeystoreType;
+import org.apache.nifi.security.util.SslContextFactory;
+import org.apache.nifi.security.util.SslContextFactory.ClientAuth;
+
+public class SSLProperties {
+
+ public static final PropertyDescriptor TRUSTSTORE = new PropertyDescriptor.Builder()
+ .name("Truststore Filename")
+ .description("The fully-qualified filename of the Truststore")
+ .defaultValue(null)
+ .addValidator(StandardValidators.FILE_EXISTS_VALIDATOR)
+ .sensitive(false)
+ .build();
+
+ public static final PropertyDescriptor TRUSTSTORE_TYPE = new PropertyDescriptor.Builder()
+ .name("Truststore Type")
+ .description("The Type of the Truststore. Either JKS or PKCS12")
+ .allowableValues("JKS", "PKCS12")
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .defaultValue(null)
+ .sensitive(false)
+ .build();
+
+ public static final PropertyDescriptor TRUSTSTORE_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Truststore Password")
+ .description("The password for the Truststore")
+ .defaultValue(null)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .sensitive(true)
+ .build();
+
+ public static final PropertyDescriptor KEYSTORE = new PropertyDescriptor.Builder()
+ .name("Keystore Filename")
+ .description("The fully-qualified filename of the Keystore")
+ .defaultValue(null)
+ .addValidator(StandardValidators.FILE_EXISTS_VALIDATOR)
+ .sensitive(false)
+ .build();
+
+ public static final PropertyDescriptor KEYSTORE_TYPE = new PropertyDescriptor.Builder()
+ .name("Keystore Type")
+ .description("The Type of the Keystore")
+ .allowableValues("JKS", "PKCS12")
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .sensitive(false)
+ .build();
+
+ public static final PropertyDescriptor KEYSTORE_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Keystore Password")
+ .defaultValue(null)
+ .description("The password for the Keystore")
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .sensitive(true)
+ .build();
+
+ public static Collection<ValidationResult> validateStore(final Map<PropertyDescriptor, String> properties) {
+ final Collection<ValidationResult> results = new ArrayList<>();
+ results.addAll(validateStore(properties, KeystoreValidationGroup.KEYSTORE));
+ results.addAll(validateStore(properties, KeystoreValidationGroup.TRUSTSTORE));
+ return results;
+ }
+
+ public static Collection<ValidationResult> validateStore(final Map<PropertyDescriptor, String> properties, final KeystoreValidationGroup keyStoreOrTrustStore) {
+ final Collection<ValidationResult> results = new ArrayList<>();
+
+ final String filename;
+ final String password;
+ final String type;
+
+ if (keyStoreOrTrustStore == KeystoreValidationGroup.KEYSTORE) {
+ filename = properties.get(KEYSTORE);
+ password = properties.get(KEYSTORE_PASSWORD);
+ type = properties.get(KEYSTORE_TYPE);
+ } else {
+ filename = properties.get(TRUSTSTORE);
+ password = properties.get(TRUSTSTORE_PASSWORD);
+ type = properties.get(TRUSTSTORE_TYPE);
+ }
+
+ final String keystoreDesc = (keyStoreOrTrustStore == KeystoreValidationGroup.KEYSTORE) ? "Keystore" : "Truststore";
+
+ final int nulls = countNulls(filename, password, type);
+ if (nulls != 3 && nulls != 0) {
+ results.add(new ValidationResult.Builder().valid(false).explanation("Must set either 0 or 3 properties for " + keystoreDesc).subject(keystoreDesc + " Properties").build());
+ } else if (nulls == 0) {
+ // all properties were filled in.
+ final File file = new File(filename);
+ if (!file.exists() || !file.canRead()) {
+ results.add(new ValidationResult.Builder().valid(false).subject(keystoreDesc + " Properties").explanation("Cannot access file " + file.getAbsolutePath()).build());
+ } else {
+ try {
+ final boolean storeValid = CertificateUtils.isStoreValid(file.toURI().toURL(), KeystoreType.valueOf(type), password.toCharArray());
+ if (!storeValid) {
+ results.add(
+ new ValidationResult.Builder()
+ .subject(keystoreDesc + " Properties")
+ .valid(false)
+ .explanation("Invalid KeyStore Password or Type specified for file " + filename)
+ .build()
+ );
+ }
+ } catch (MalformedURLException e) {
+ results.add(new ValidationResult.Builder().subject(keystoreDesc + " Properties").valid(false).explanation("Malformed URL from file: " + e).build());
+ }
+ }
+ }
+
+ return results;
+ }
+
+ private static int countNulls(Object... objects) {
+ int count = 0;
+ for (final Object x : objects) {
+ if (x == null) {
+ count++;
+ }
+ }
+
+ return count;
+ }
+
+ public static enum KeystoreValidationGroup {
+
+ KEYSTORE, TRUSTSTORE
+ }
+
+ private static final String DEFAULT_SSL_PROTOCOL_ALGORITHM = "TLS";
+
+ public static List<PropertyDescriptor> getKeystoreDescriptors(final boolean required) {
+ final List<PropertyDescriptor> descriptors = new ArrayList<>();
+ for (final PropertyDescriptor descriptor : KEYSTORE_DESCRIPTORS) {
+ final PropertyDescriptor.Builder builder = new PropertyDescriptor.Builder().fromPropertyDescriptor(descriptor).required(required);
+ if (required && descriptor.getName().equals(KEYSTORE_TYPE.getName())) {
+ builder.defaultValue("JKS");
+ }
+ descriptors.add(builder.build());
+ }
+
+ return descriptors;
+ }
+
+ public static List<PropertyDescriptor> getTruststoreDescriptors(final boolean required) {
+ final List<PropertyDescriptor> descriptors = new ArrayList<>();
+ for (final PropertyDescriptor descriptor : TRUSTSTORE_DESCRIPTORS) {
+ final PropertyDescriptor.Builder builder = new PropertyDescriptor.Builder().fromPropertyDescriptor(descriptor).required(required);
+ if (required && descriptor.getName().equals(TRUSTSTORE_TYPE.getName())) {
+ builder.defaultValue("JKS");
+ }
+ descriptors.add(builder.build());
+ }
+
+ return descriptors;
+ }
+
+ public static SSLContext createSSLContext(final ProcessContext context, final ClientAuth clientAuth)
+ throws UnrecoverableKeyException, KeyManagementException, KeyStoreException, NoSuchAlgorithmException, CertificateException, IOException {
+ final String keystoreFile = context.getProperty(KEYSTORE).getValue();
+ if (keystoreFile == null) {
+ return SslContextFactory.createTrustSslContext(
+ context.getProperty(TRUSTSTORE).getValue(),
+ context.getProperty(TRUSTSTORE_PASSWORD).getValue().toCharArray(),
+ context.getProperty(TRUSTSTORE_TYPE).getValue(),
+ DEFAULT_SSL_PROTOCOL_ALGORITHM);
+ } else {
+ final String truststoreFile = context.getProperty(TRUSTSTORE).getValue();
+ if (truststoreFile == null) {
+ return SslContextFactory.createSslContext(
+ context.getProperty(KEYSTORE).getValue(),
+ context.getProperty(KEYSTORE_PASSWORD).getValue().toCharArray(),
+ context.getProperty(KEYSTORE_TYPE).getValue(), DEFAULT_SSL_PROTOCOL_ALGORITHM);
+ } else {
+ return SslContextFactory.createSslContext(
+ context.getProperty(KEYSTORE).getValue(),
+ context.getProperty(KEYSTORE_PASSWORD).getValue().toCharArray(),
+ context.getProperty(KEYSTORE_TYPE).getValue(),
+ context.getProperty(TRUSTSTORE).getValue(),
+ context.getProperty(TRUSTSTORE_PASSWORD).getValue().toCharArray(),
+ context.getProperty(TRUSTSTORE_TYPE).getValue(),
+ clientAuth,
+ DEFAULT_SSL_PROTOCOL_ALGORITHM);
+ }
+ }
+ }
+
+ private static final Set<PropertyDescriptor> KEYSTORE_DESCRIPTORS = new HashSet<>();
+ private static final Set<PropertyDescriptor> TRUSTSTORE_DESCRIPTORS = new HashSet<>();
+
+ static {
+ KEYSTORE_DESCRIPTORS.add(KEYSTORE);
+ KEYSTORE_DESCRIPTORS.add(KEYSTORE_TYPE);
+ KEYSTORE_DESCRIPTORS.add(KEYSTORE_PASSWORD);
+
+ TRUSTSTORE_DESCRIPTORS.add(TRUSTSTORE);
+ TRUSTSTORE_DESCRIPTORS.add(TRUSTSTORE_TYPE);
+ TRUSTSTORE_DESCRIPTORS.add(TRUSTSTORE_PASSWORD);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-processor-utilities/src/main/java/org/apache/nifi/processor/util/StandardValidators.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-processor-utilities/src/main/java/org/apache/nifi/processor/util/StandardValidators.java b/nifi-commons/nifi-processor-utilities/src/main/java/org/apache/nifi/processor/util/StandardValidators.java
new file mode 100644
index 0000000..37ba7d8
--- /dev/null
+++ b/nifi-commons/nifi-processor-utilities/src/main/java/org/apache/nifi/processor/util/StandardValidators.java
@@ -0,0 +1,708 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor.util;
+
+import java.io.File;
+import java.net.URI;
+import java.net.URL;
+import java.nio.charset.Charset;
+import java.nio.charset.UnsupportedCharsetException;
+import java.util.Collection;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Pattern;
+
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.components.Validator;
+import org.apache.nifi.controller.ControllerService;
+import org.apache.nifi.expression.AttributeExpression.ResultType;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.DataUnit;
+import org.apache.nifi.util.FormatUtils;
+
+public class StandardValidators {
+
+ //
+ //
+ // STATICALLY DEFINED VALIDATORS
+ //
+ //
+ public static final Validator ATTRIBUTE_KEY_VALIDATOR = new Validator() {
+ @Override
+ public ValidationResult validate(final String subject, final String input, final ValidationContext context) {
+ final ValidationResult.Builder builder = new ValidationResult.Builder();
+ builder.subject(subject).input(input);
+ if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(input)) {
+ return builder.valid(true).explanation("Contains Expression Language").build();
+ }
+
+ try {
+ FlowFile.KeyValidator.validateKey(input);
+ builder.valid(true);
+ } catch (final IllegalArgumentException e) {
+ builder.valid(false).explanation(e.getMessage());
+ }
+
+ return builder.build();
+ }
+ };
+
+ public static final Validator ATTRIBUTE_KEY_PROPERTY_NAME_VALIDATOR = new Validator() {
+ @Override
+ public ValidationResult validate(final String subject, final String input, final ValidationContext context) {
+ final ValidationResult.Builder builder = new ValidationResult.Builder();
+ builder.subject("Property Name").input(subject);
+ if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(input)) {
+ return builder.valid(true).explanation("Contains Expression Language").build();
+ }
+
+ try {
+ FlowFile.KeyValidator.validateKey(subject);
+ builder.valid(true);
+ } catch (final IllegalArgumentException e) {
+ builder.valid(false).explanation(e.getMessage());
+ }
+
+ return builder.build();
+ }
+ };
+
+ public static final Validator POSITIVE_INTEGER_VALIDATOR = new Validator() {
+ @Override
+ public ValidationResult validate(final String subject, final String value, final ValidationContext context) {
+ if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(value)) {
+ return new ValidationResult.Builder().subject(subject).input(value).explanation("Expression Language Present").valid(true).build();
+ }
+
+ String reason = null;
+ try {
+ final int intVal = Integer.parseInt(value);
+
+ if (intVal <= 0) {
+ reason = "not a positive value";
+ }
+ } catch (final NumberFormatException e) {
+ reason = "not a valid integer";
+ }
+
+ return new ValidationResult.Builder().subject(subject).input(value).explanation(reason).valid(reason == null).build();
+ }
+ };
+
+ public static final Validator POSITIVE_LONG_VALIDATOR = new Validator() {
+ @Override
+ public ValidationResult validate(final String subject, final String value, final ValidationContext context) {
+ if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(value)) {
+ return new ValidationResult.Builder().subject(subject).input(value).explanation("Expression Language Present").valid(true).build();
+ }
+
+ String reason = null;
+ try {
+ final long longVal = Long.parseLong(value);
+
+ if (longVal <= 0) {
+ reason = "not a positive value";
+ }
+ } catch (final NumberFormatException e) {
+ reason = "not a valid 64-bit integer";
+ }
+
+ return new ValidationResult.Builder().subject(subject).input(value).explanation(reason).valid(reason == null).build();
+ }
+ };
+
+ public static final Validator PORT_VALIDATOR = createLongValidator(1, 65535, true);
+
+ public static final Validator NON_EMPTY_VALIDATOR = new Validator() {
+ @Override
+ public ValidationResult validate(final String subject, final String value, final ValidationContext context) {
+ return new ValidationResult.Builder().subject(subject).input(value).valid(value != null && !value.isEmpty()).explanation(subject + " cannot be empty").build();
+ }
+ };
+
+ public static final Validator BOOLEAN_VALIDATOR = new Validator() {
+ @Override
+ public ValidationResult validate(final String subject, final String value, final ValidationContext context) {
+ if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(value)) {
+ return new ValidationResult.Builder().subject(subject).input(value).explanation("Expression Language Present").valid(true).build();
+ }
+
+ final boolean valid = "true".equalsIgnoreCase(value) || "false".equalsIgnoreCase(value);
+ final String explanation = valid ? null : "Value must be 'true' or 'false'";
+ return new ValidationResult.Builder().subject(subject).input(value).valid(valid).explanation(explanation).build();
+ }
+ };
+
+ public static final Validator INTEGER_VALIDATOR = new Validator() {
+ @Override
+ public ValidationResult validate(final String subject, final String value, final ValidationContext context) {
+ if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(value)) {
+ return new ValidationResult.Builder().subject(subject).input(value).explanation("Expression Language Present").valid(true).build();
+ }
+
+ String reason = null;
+ try {
+ Integer.parseInt(value);
+ } catch (final NumberFormatException e) {
+ reason = "not a valid integer";
+ }
+
+ return new ValidationResult.Builder().subject(subject).input(value).explanation(reason).valid(reason == null).build();
+ }
+ };
+
+ public static final Validator LONG_VALIDATOR = new Validator() {
+ @Override
+ public ValidationResult validate(final String subject, final String value, final ValidationContext context) {
+ if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(value)) {
+ return new ValidationResult.Builder().subject(subject).input(value).explanation("Expression Language Present").valid(true).build();
+ }
+
+ String reason = null;
+ try {
+ Long.parseLong(value);
+ } catch (final NumberFormatException e) {
+ reason = "not a valid Long";
+ }
+
+ return new ValidationResult.Builder().subject(subject).input(value).explanation(reason).valid(reason == null).build();
+ }
+ };
+
+ public static final Validator NON_NEGATIVE_INTEGER_VALIDATOR = new Validator() {
+ @Override
+ public ValidationResult validate(final String subject, final String value, final ValidationContext context) {
+ if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(value)) {
+ return new ValidationResult.Builder().subject(subject).input(value).explanation("Expression Language Present").valid(true).build();
+ }
+
+ String reason = null;
+ try {
+ final int intVal = Integer.parseInt(value);
+
+ if (intVal < 0) {
+ reason = "value is negative";
+ }
+ } catch (final NumberFormatException e) {
+ reason = "value is not a valid integer";
+ }
+
+ return new ValidationResult.Builder().subject(subject).input(value).explanation(reason).valid(reason == null).build();
+ }
+ };
+
+ public static final Validator CHARACTER_SET_VALIDATOR = new Validator() {
+ @Override
+ public ValidationResult validate(final String subject, final String value, final ValidationContext context) {
+ if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(value)) {
+ return new ValidationResult.Builder().subject(subject).input(value).explanation("Expression Language Present").valid(true).build();
+ }
+
+ String reason = null;
+ try {
+ if (!Charset.isSupported(value)) {
+ reason = "Character Set is not supported by this JVM.";
+ }
+ } catch (final UnsupportedCharsetException uce) {
+ reason = "Character Set is not supported by this JVM.";
+ } catch (final IllegalArgumentException iae) {
+ reason = "Character Set value cannot be null.";
+ }
+
+ return new ValidationResult.Builder().subject(subject).input(value).explanation(reason).valid(reason == null).build();
+ }
+ };
+
+ /**
+ * URL Validator that does not allow the Expression Language to be used
+ */
+ public static final Validator URL_VALIDATOR = createURLValidator();
+
+ public static final Validator URI_VALIDATOR = new Validator() {
+ @Override
+ public ValidationResult validate(final String subject, final String input, final ValidationContext context) {
+ if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(input)) {
+ return new ValidationResult.Builder().subject(subject).input(input).explanation("Expression Language Present").valid(true).build();
+ }
+
+ try {
+ new URI(input);
+ return new ValidationResult.Builder().subject(subject).input(input).explanation("Valid URI").valid(true).build();
+ } catch (final Exception e) {
+ return new ValidationResult.Builder().subject(subject).input(input).explanation("Not a valid URI").valid(false).build();
+ }
+ }
+ };
+
+ public static final Validator REGULAR_EXPRESSION_VALIDATOR = createRegexValidator(0, Integer.MAX_VALUE, false);
+
+ public static final Validator ATTRIBUTE_EXPRESSION_LANGUAGE_VALIDATOR = new Validator() {
+ @Override
+ public ValidationResult validate(final String subject, final String input, final ValidationContext context) {
+ if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(input)) {
+ return new ValidationResult.Builder().subject(subject).input(input).explanation("Expression Language Present").valid(true).build();
+ }
+
+ try {
+ context.newExpressionLanguageCompiler().compile(input);
+ return new ValidationResult.Builder().subject(subject).input(input).valid(true).build();
+ } catch (final Exception e) {
+ return new ValidationResult.Builder().subject(subject).input(input).valid(false).explanation(e.getMessage()).build();
+ }
+ }
+
+ };
+
+ public static final Validator TIME_PERIOD_VALIDATOR = new Validator() {
+ @Override
+ public ValidationResult validate(final String subject, final String input, final ValidationContext context) {
+ if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(input)) {
+ return new ValidationResult.Builder().subject(subject).input(input).explanation("Expression Language Present").valid(true).build();
+ }
+
+ if (input == null) {
+ return new ValidationResult.Builder().subject(subject).input(input).valid(false).explanation("Time Period cannot be null").build();
+ }
+ if (Pattern.compile(FormatUtils.TIME_DURATION_REGEX).matcher(input.toLowerCase()).matches()) {
+ return new ValidationResult.Builder().subject(subject).input(input).valid(true).build();
+ } else {
+ return new ValidationResult.Builder()
+ .subject(subject)
+ .input(input)
+ .valid(false)
+ .explanation("Must be of format <duration> <TimeUnit> where <duration> is a "
+ + "non-negative integer and TimeUnit is a supported Time Unit, such "
+ + "as: nanos, millis, secs, mins, hrs, days")
+ .build();
+ }
+ }
+ };
+
+ public static final Validator DATA_SIZE_VALIDATOR = new Validator() {
+ @Override
+ public ValidationResult validate(final String subject, final String input, final ValidationContext context) {
+ if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(input)) {
+ return new ValidationResult.Builder().subject(subject).input(input).explanation("Expression Language Present").valid(true).build();
+ }
+
+ if (input == null) {
+ return new ValidationResult.Builder()
+ .subject(subject)
+ .input(input)
+ .valid(false)
+ .explanation("Data Size cannot be null")
+ .build();
+ }
+ if (Pattern.compile(DataUnit.DATA_SIZE_REGEX).matcher(input.toUpperCase()).matches()) {
+ return new ValidationResult.Builder().subject(subject).input(input).valid(true).build();
+ } else {
+ return new ValidationResult.Builder()
+ .subject(subject).input(input)
+ .valid(false)
+ .explanation("Must be of format <Data Size> <Data Unit> where <Data Size>"
+ + " is a non-negative integer and <Data Unit> is a supported Data"
+ + " Unit, such as: B, KB, MB, GB, TB")
+ .build();
+ }
+ }
+ };
+
+ public static final Validator FILE_EXISTS_VALIDATOR = new FileExistsValidator(true);
+
+ //
+ //
+ // FACTORY METHODS FOR VALIDATORS
+ //
+ //
+ public static Validator createDirectoryExistsValidator(final boolean allowExpressionLanguage, final boolean createDirectoryIfMissing) {
+ return new DirectoryExistsValidator(allowExpressionLanguage, createDirectoryIfMissing);
+ }
+
+ private static Validator createURLValidator() {
+ return new Validator() {
+ @Override
+ public ValidationResult validate(final String subject, final String input, final ValidationContext context) {
+ if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(input)) {
+ return new ValidationResult.Builder().subject(subject).input(input).explanation("Expression Language Present").valid(true).build();
+ }
+
+ try {
+ final String evaluatedInput = context.newPropertyValue(input).evaluateAttributeExpressions().getValue();
+ new URL(evaluatedInput);
+ return new ValidationResult.Builder().subject(subject).input(input).explanation("Valid URL").valid(true).build();
+ } catch (final Exception e) {
+ return new ValidationResult.Builder().subject(subject).input(input).explanation("Not a valid URL").valid(false).build();
+ }
+ }
+ };
+ }
+
+ public static Validator createTimePeriodValidator(final long minTime, final TimeUnit minTimeUnit, final long maxTime, final TimeUnit maxTimeUnit) {
+ return new TimePeriodValidator(minTime, minTimeUnit, maxTime, maxTimeUnit);
+ }
+
+ public static Validator createAttributeExpressionLanguageValidator(final ResultType expectedResultType) {
+ return createAttributeExpressionLanguageValidator(expectedResultType, true);
+ }
+
+ public static Validator createDataSizeBoundsValidator(final long minBytesInclusive, final long maxBytesInclusive) {
+ return new Validator() {
+
+ @Override
+ public ValidationResult validate(final String subject, final String input, final ValidationContext context) {
+ if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(input)) {
+ return new ValidationResult.Builder().subject(subject).input(input).explanation("Expression Language Present").valid(true).build();
+ }
+
+ final ValidationResult vr = DATA_SIZE_VALIDATOR.validate(subject, input, context);
+ if (!vr.isValid()) {
+ return vr;
+ }
+ final long dataSizeBytes = DataUnit.parseDataSize(input, DataUnit.B).longValue();
+ if (dataSizeBytes < minBytesInclusive) {
+ return new ValidationResult.Builder().subject(subject).input(input).valid(false).explanation("Cannot be smaller than " + minBytesInclusive + " bytes").build();
+ }
+ if (dataSizeBytes > maxBytesInclusive) {
+ return new ValidationResult.Builder().subject(subject).input(input).valid(false).explanation("Cannot be larger than " + maxBytesInclusive + " bytes").build();
+ }
+ return new ValidationResult.Builder().subject(subject).input(input).valid(true).build();
+ }
+ };
+
+ }
+
+ public static Validator createRegexMatchingValidator(final Pattern pattern) {
+ return new Validator() {
+ @Override
+ public ValidationResult validate(final String subject, final String input, final ValidationContext context) {
+ if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(input)) {
+ return new ValidationResult.Builder().subject(subject).input(input).explanation("Expression Language Present").valid(true).build();
+ }
+
+ final boolean matches = pattern.matcher(input).matches();
+ return new ValidationResult.Builder()
+ .input(input)
+ .subject(subject)
+ .valid(matches)
+ .explanation(matches ? null : "Value does not match regular expression: " + pattern.pattern())
+ .build();
+ }
+ };
+ }
+
+ /**
+ * Creates a @{link Validator} that ensure that a value is a valid Java
+ * Regular Expression with at least <code>minCapturingGroups</code>
+ * capturing groups and at most <code>maxCapturingGroups</code> capturing
+ * groups. If <code>supportAttributeExpressionLanguage</code> is set to
+ * <code>true</code>, the value may also include the Expression Language,
+ * but the result of evaluating the Expression Language will be applied
+ * before the Regular Expression is performed. In this case, the Expression
+ * Language will not support FlowFile Attributes but only System/JVM
+ * Properties
+ *
+ * @param minCapturingGroups minimum capturing groups allowed
+ * @param maxCapturingGroups maximum capturing groups allowed
+ * @param supportAttributeExpressionLanguage whether or not to support
+ * expression language
+ * @return validator
+ */
+ public static Validator createRegexValidator(final int minCapturingGroups, final int maxCapturingGroups, final boolean supportAttributeExpressionLanguage) {
+ return new Validator() {
+ @Override
+ public ValidationResult validate(final String subject, final String value, final ValidationContext context) {
+ try {
+ final String substituted;
+ if (supportAttributeExpressionLanguage) {
+ try {
+ substituted = context.newPropertyValue(value).evaluateAttributeExpressions().getValue();
+ } catch (final Exception e) {
+ return new ValidationResult.Builder()
+ .subject(subject)
+ .input(value)
+ .valid(false)
+ .explanation("Failed to evaluate the Attribute Expression Language due to " + e.toString())
+ .build();
+ }
+ } else {
+ substituted = value;
+ }
+
+ final Pattern pattern = Pattern.compile(substituted);
+ final int numGroups = pattern.matcher("").groupCount();
+ if (numGroups < minCapturingGroups || numGroups > maxCapturingGroups) {
+ return new ValidationResult.Builder()
+ .subject(subject)
+ .input(value)
+ .valid(false)
+ .explanation("RegEx is required to have between " + minCapturingGroups + " and " + maxCapturingGroups + " Capturing Groups but has " + numGroups)
+ .build();
+ }
+
+ return new ValidationResult.Builder().subject(subject).input(value).valid(true).build();
+ } catch (final Exception e) {
+ return new ValidationResult.Builder()
+ .subject(subject)
+ .input(value)
+ .valid(false)
+ .explanation("Not a valid Java Regular Expression")
+ .build();
+ }
+
+ }
+ };
+ }
+
+ public static Validator createAttributeExpressionLanguageValidator(final ResultType expectedResultType, final boolean allowExtraCharacters) {
+ return new Validator() {
+ @Override
+ public ValidationResult validate(final String subject, final String input, final ValidationContext context) {
+ final String syntaxError = context.newExpressionLanguageCompiler().validateExpression(input, allowExtraCharacters);
+ if (syntaxError != null) {
+ return new ValidationResult.Builder().subject(subject).input(input).valid(false).explanation(syntaxError).build();
+ }
+
+ final ResultType resultType = allowExtraCharacters ? ResultType.STRING : context.newExpressionLanguageCompiler().getResultType(input);
+ if (!resultType.equals(expectedResultType)) {
+ return new ValidationResult.Builder()
+ .subject(subject)
+ .input(input)
+ .valid(false)
+ .explanation("Expected Attribute Query to return type " + expectedResultType + " but query returns type " + resultType)
+ .build();
+ }
+
+ return new ValidationResult.Builder().subject(subject).input(input).valid(true).build();
+ }
+ };
+ }
+
+ public static Validator createLongValidator(final long minimum, final long maximum, final boolean inclusive) {
+ return new Validator() {
+ @Override
+ public ValidationResult validate(final String subject, final String input, final ValidationContext context) {
+ if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(input)) {
+ return new ValidationResult.Builder().subject(subject).input(input).explanation("Expression Language Present").valid(true).build();
+ }
+
+ String reason = null;
+ try {
+ final long longVal = Long.parseLong(input);
+ if (longVal < minimum || (!inclusive && longVal == minimum) | longVal > maximum || (!inclusive && longVal == maximum)) {
+ reason = "Value must be between " + minimum + " and " + maximum + " (" + (inclusive ? "inclusive" : "exclusive") + ")";
+ }
+ } catch (final NumberFormatException e) {
+ reason = "not a valid integer";
+ }
+
+ return new ValidationResult.Builder().subject(subject).input(input).explanation(reason).valid(reason == null).build();
+ }
+
+ };
+ }
+
+ //
+ //
+ // SPECIFIC VALIDATOR IMPLEMENTATIONS THAT CANNOT BE ANONYMOUS CLASSES
+ //
+ //
+ static class TimePeriodValidator implements Validator {
+
+ private final Pattern pattern;
+
+ private final long minNanos;
+ private final long maxNanos;
+
+ private final String minValueEnglish;
+ private final String maxValueEnglish;
+
+ public TimePeriodValidator(final long minValue, final TimeUnit minTimeUnit, final long maxValue, final TimeUnit maxTimeUnit) {
+ pattern = Pattern.compile(FormatUtils.TIME_DURATION_REGEX);
+
+ this.minNanos = TimeUnit.NANOSECONDS.convert(minValue, minTimeUnit);
+ this.maxNanos = TimeUnit.NANOSECONDS.convert(maxValue, maxTimeUnit);
+ this.minValueEnglish = minValue + " " + minTimeUnit.toString();
+ this.maxValueEnglish = maxValue + " " + maxTimeUnit.toString();
+ }
+
+ @Override
+ public ValidationResult validate(final String subject, final String input, final ValidationContext context) {
+ if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(input)) {
+ return new ValidationResult.Builder().subject(subject).input(input).explanation("Expression Language Present").valid(true).build();
+ }
+
+ if (input == null) {
+ return new ValidationResult.Builder().subject(subject).input(input).valid(false).explanation("Time Period cannot be null").build();
+ }
+ final String lowerCase = input.toLowerCase();
+ final boolean validSyntax = pattern.matcher(lowerCase).matches();
+ final ValidationResult.Builder builder = new ValidationResult.Builder();
+ if (validSyntax) {
+ final long nanos = FormatUtils.getTimeDuration(lowerCase, TimeUnit.NANOSECONDS);
+
+ if (nanos < minNanos || nanos > maxNanos) {
+ builder.subject(subject).input(input).valid(false)
+ .explanation("Must be in the range of " + minValueEnglish + " to " + maxValueEnglish);
+ } else {
+ builder.subject(subject).input(input).valid(true);
+ }
+ } else {
+ builder.subject(subject).input(input).valid(false)
+ .explanation("Must be of format <duration> <TimeUnit> where <duration> is a non-negative "
+ + "integer and TimeUnit is a supported Time Unit, such as: nanos, millis, secs, mins, hrs, days");
+ }
+ return builder.build();
+ }
+ }
+
+ public static class FileExistsValidator implements Validator {
+
+ private final boolean allowEL;
+
+ public FileExistsValidator(final boolean allowExpressionLanguage) {
+ this.allowEL = allowExpressionLanguage;
+ }
+
+ @Override
+ public ValidationResult validate(final String subject, final String value, final ValidationContext context) {
+ if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(value)) {
+ return new ValidationResult.Builder().subject(subject).input(value).explanation("Expression Language Present").valid(true).build();
+ }
+
+ final String substituted;
+ if (allowEL) {
+ try {
+ substituted = context.newPropertyValue(value).evaluateAttributeExpressions().getValue();
+ } catch (final Exception e) {
+ return new ValidationResult.Builder().subject(subject).input(value).valid(false)
+ .explanation("Not a valid Expression Language value: " + e.getMessage()).build();
+ }
+ } else {
+ substituted = value;
+ }
+
+ final File file = new File(substituted);
+ final boolean valid = file.exists();
+ final String explanation = valid ? null : "File " + file + " does not exist";
+ return new ValidationResult.Builder().subject(subject).input(value).valid(valid).explanation(explanation).build();
+ }
+ }
+
+ public static class DirectoryExistsValidator implements Validator {
+
+ private final boolean allowEL;
+ private final boolean create;
+
+ public DirectoryExistsValidator(final boolean allowExpressionLanguage, final boolean create) {
+ this.allowEL = allowExpressionLanguage;
+ this.create = create;
+ }
+
+ @Override
+ public ValidationResult validate(final String subject, final String value, final ValidationContext context) {
+ if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(value)) {
+ return new ValidationResult.Builder().subject(subject).input(value).explanation("Expression Language Present").valid(true).build();
+ }
+
+ final String substituted;
+ if (allowEL) {
+ try {
+ substituted = context.newPropertyValue(value).evaluateAttributeExpressions().getValue();
+ } catch (final Exception e) {
+ return new ValidationResult.Builder().subject(subject).input(value).valid(false)
+ .explanation("Not a valid Expression Language value: " + e.getMessage()).build();
+ }
+
+ if (substituted.trim().isEmpty() && !value.trim().isEmpty()) {
+ // User specified an Expression and nothing more... assume valid.
+ return new ValidationResult.Builder().subject(subject).input(value).valid(true).build();
+ }
+ } else {
+ substituted = value;
+ }
+
+ String reason = null;
+ try {
+ final File file = new File(substituted);
+ if (!file.exists()) {
+ if (!create) {
+ reason = "Directory does not exist";
+ } else if (!file.mkdirs()) {
+ reason = "Directory does not exist and could not be created";
+ }
+ } else if (!file.isDirectory()) {
+ reason = "Path does not point to a directory";
+ }
+ } catch (final Exception e) {
+ reason = "Value is not a valid directory name";
+ }
+
+ return new ValidationResult.Builder().subject(subject).input(value).explanation(reason).valid(reason == null).build();
+ }
+ }
+
+ /**
+ * Creates a validator based on existence of a {@link ControllerService}.
+ *
+ * @param serviceClass the controller service API your
+ * {@link ConfigurableComponent} depends on
+ * @return a Validator
+ * @deprecated As of release 0.1.0-incubating, replaced by
+ * {@link org.apache.nifi.components.PropertyDescriptor.Builder#identifiesControllerService(Class)}
+ */
+ @Deprecated
+ public static Validator createControllerServiceExistsValidator(final Class<? extends ControllerService> serviceClass) {
+ return new Validator() {
+ @Override
+ public ValidationResult validate(final String subject, final String input, final ValidationContext context) {
+ if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(input)) {
+ return new ValidationResult.Builder().subject(subject).input(input).explanation("Expression Language Present").valid(true).build();
+ }
+
+ final ControllerService svc = context.getControllerServiceLookup().getControllerService(input);
+
+ if (svc == null) {
+ return new ValidationResult.Builder().valid(false).input(input).subject(subject).explanation("No Controller Service exists with this ID").build();
+ }
+
+ if (!serviceClass.isAssignableFrom(svc.getClass())) {
+ return new ValidationResult.Builder()
+ .valid(false)
+ .input(input)
+ .subject(subject)
+ .explanation("Controller Service with this ID is of type " + svc.getClass().getName() + " but is expected to be of type " + serviceClass.getName())
+ .build();
+ }
+
+ final ValidationContext serviceValidationContext = context.getControllerServiceValidationContext(svc);
+ final Collection<ValidationResult> serviceValidationResults = svc.validate(serviceValidationContext);
+ for (final ValidationResult result : serviceValidationResults) {
+ if (!result.isValid()) {
+ return new ValidationResult.Builder()
+ .valid(false)
+ .input(input)
+ .subject(subject)
+ .explanation("Controller Service " + input + " is not valid: " + result.getExplanation())
+ .build();
+ }
+ }
+
+ return new ValidationResult.Builder().input(input).subject(subject).valid(true).build();
+ }
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-processor-utilities/src/test/java/org/apache/nifi/processor/TestFormatUtils.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-processor-utilities/src/test/java/org/apache/nifi/processor/TestFormatUtils.java b/nifi-commons/nifi-processor-utilities/src/test/java/org/apache/nifi/processor/TestFormatUtils.java
new file mode 100644
index 0000000..359def2
--- /dev/null
+++ b/nifi-commons/nifi-processor-utilities/src/test/java/org/apache/nifi/processor/TestFormatUtils.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.nifi.util.FormatUtils;
+
+import org.junit.Test;
+
+public class TestFormatUtils {
+
+ @Test
+ public void testParse() {
+ assertEquals(3, FormatUtils.getTimeDuration("3000 ms", TimeUnit.SECONDS));
+ assertEquals(3000, FormatUtils.getTimeDuration("3000 s", TimeUnit.SECONDS));
+ assertEquals(0, FormatUtils.getTimeDuration("999 millis", TimeUnit.SECONDS));
+ assertEquals(4L * 24L * 60L * 60L * 1000000000L, FormatUtils.getTimeDuration("4 days", TimeUnit.NANOSECONDS));
+ assertEquals(24, FormatUtils.getTimeDuration("1 DAY", TimeUnit.HOURS));
+ assertEquals(60, FormatUtils.getTimeDuration("1 hr", TimeUnit.MINUTES));
+ assertEquals(60, FormatUtils.getTimeDuration("1 Hrs", TimeUnit.MINUTES));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-processor-utilities/src/test/java/org/apache/nifi/processor/util/TestStandardValidators.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-processor-utilities/src/test/java/org/apache/nifi/processor/util/TestStandardValidators.java b/nifi-commons/nifi-processor-utilities/src/test/java/org/apache/nifi/processor/util/TestStandardValidators.java
new file mode 100644
index 0000000..bcd402d
--- /dev/null
+++ b/nifi-commons/nifi-processor-utilities/src/test/java/org/apache/nifi/processor/util/TestStandardValidators.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processor.util;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.components.Validator;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+public class TestStandardValidators {
+
+ @Test
+ public void testTimePeriodValidator() {
+ Validator val = StandardValidators.createTimePeriodValidator(1L, TimeUnit.SECONDS, Long.MAX_VALUE, TimeUnit.NANOSECONDS);
+ ValidationResult vr;
+
+ final ValidationContext validationContext = Mockito.mock(ValidationContext.class);
+
+ vr = val.validate("TimePeriodTest", "0 sense made", validationContext);
+ assertFalse(vr.isValid());
+
+ vr = val.validate("TimePeriodTest", null, validationContext);
+ assertFalse(vr.isValid());
+
+ vr = val.validate("TimePeriodTest", "0 secs", validationContext);
+ assertFalse(vr.isValid());
+
+ vr = val.validate("TimePeriodTest", "999 millis", validationContext);
+ assertFalse(vr.isValid());
+
+ vr = val.validate("TimePeriodTest", "999999999 nanos", validationContext);
+ assertFalse(vr.isValid());
+
+ vr = val.validate("TimePeriodTest", "1 sec", validationContext);
+ assertTrue(vr.isValid());
+ }
+
+ @Test
+ public void testDataSizeBoundsValidator() {
+ Validator val = StandardValidators.createDataSizeBoundsValidator(100, 1000);
+ ValidationResult vr;
+
+ final ValidationContext validationContext = Mockito.mock(ValidationContext.class);
+ vr = val.validate("DataSizeBounds", "5 GB", validationContext);
+ assertFalse(vr.isValid());
+
+ vr = val.validate("DataSizeBounds", "0 B", validationContext);
+ assertFalse(vr.isValid());
+
+ vr = val.validate("DataSizeBounds", "99 B", validationContext);
+ assertFalse(vr.isValid());
+
+ vr = val.validate("DataSizeBounds", "100 B", validationContext);
+ assertTrue(vr.isValid());
+
+ vr = val.validate("DataSizeBounds", "999 B", validationContext);
+ assertTrue(vr.isValid());
+
+ vr = val.validate("DataSizeBounds", "1000 B", validationContext);
+ assertTrue(vr.isValid());
+
+ vr = val.validate("DataSizeBounds", "1001 B", validationContext);
+ assertFalse(vr.isValid());
+
+ vr = val.validate("DataSizeBounds", "water", validationContext);
+ assertFalse(vr.isValid());
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-properties/.gitignore
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-properties/.gitignore b/nifi-commons/nifi-properties/.gitignore
new file mode 100755
index 0000000..073c9fa
--- /dev/null
+++ b/nifi-commons/nifi-properties/.gitignore
@@ -0,0 +1,3 @@
+/target
+/target
+/target
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-properties/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-properties/pom.xml b/nifi-commons/nifi-properties/pom.xml
new file mode 100644
index 0000000..32dbf40
--- /dev/null
+++ b/nifi-commons/nifi-properties/pom.xml
@@ -0,0 +1,24 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-commons</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>nifi-properties</artifactId>
+</project>
[44/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/provenance/ProvenanceReporter.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/provenance/ProvenanceReporter.java b/nifi-api/src/main/java/org/apache/nifi/provenance/ProvenanceReporter.java
new file mode 100644
index 0000000..da242b8
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/provenance/ProvenanceReporter.java
@@ -0,0 +1,616 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+
+import java.util.Collection;
+
+/**
+ * ProvenanceReporter generates and records Provenance-related events. A
+ * ProvenanceReporter is always tied to a {@link ProcessSession}. Any events
+ * that are generated are reported to Provenance only after the session has been
+ * committed. If the session is rolled back, the events related to that session
+ * are purged.
+ */
+public interface ProvenanceReporter {
+
+ /**
+ * Emits a Provenance Event of type
+ * {@link ProvenanceEventType#RECEIVE RECEIVE} that indicates that the given
+ * FlowFile was created from data received from an external source.
+ *
+ * @param flowFile the FlowFile that was received
+ * @param transitUri A URI that provides information about the System and
+ * Protocol information over which the transfer occurred. The intent of this
+ * field is such that both the sender and the receiver can publish the
+ * events to an external Enterprise-wide system that is then able to
+ * correlate the SEND and RECEIVE events.
+ */
+ void receive(FlowFile flowFile, String transitUri);
+
+ /**
+ * Emits a Provenance Event of type
+ * {@link ProvenanceEventType#RECEIVE RECEIVE} that indicates that the given
+ * FlowFile was created from data received from the specified URI and that
+ * the source system used the specified identifier (a URI with namespace) to
+ * refer to the data.
+ *
+ * @param flowFile the FlowFile that was received
+ * @param transitUri A URI that provides information about the System and
+ * Protocol information over which the transfer occurred. The intent of this
+ * field is such that both the sender and the receiver can publish the
+ * events to an external Enterprise-wide system that is then able to
+ * correlate the SEND and RECEIVE events.
+ * @param sourceSystemFlowFileIdentifier the URI/identifier that the source
+ * system uses to refer to the data; if this value is non-null and is not a
+ * URI, the prefix "urn:tdo:" will be used to form a URI.
+ */
+ void receive(FlowFile flowFile, String transitUri, String sourceSystemFlowFileIdentifier);
+
+ /**
+ * Emits a Provenance Event of type
+ * {@link ProvenanceEventType#RECEIVE RECEIVE} that indicates that the given
+ * FlowFile was created from data received from an external source.
+ *
+ * @param flowFile the FlowFile that was received
+ * @param transitUri A URI that provides information about the System and
+ * Protocol information over which the transfer occurred. The intent of this
+ * field is such that both the sender and the receiver can publish the
+ * events to an external Enterprise-wide system that is then able to
+ * correlate the SEND and RECEIVE events.
+ * @param transmissionMillis the number of milliseconds taken to transfer
+ * the data
+ */
+ void receive(FlowFile flowFile, String transitUri, long transmissionMillis);
+
+ /**
+ * Emits a Provenance Event of type
+ * {@link ProvenanceEventType#RECEIVE RECEIVE} that indicates that the given
+ * FlowFile was created from data received from an external source and
+ * provides additional details about the receipt of the FlowFile, such as a
+ * remote system's Distinguished Name.
+ *
+ * @param flowFile the FlowFile that was received
+ * @param transitUri A URI that provides information about the System and
+ * Protocol information over which the transfer occurred. The intent of this
+ * field is such that both the sender and the receiver can publish the
+ * events to an external Enterprise-wide system that is then able to
+ * correlate the SEND and RECEIVE events.
+ * @param details details about the receive event; for example, it may be
+ * relevant to include the DN of the sending system
+ * @param transmissionMillis the number of milliseconds taken to transfer
+ * the data
+ */
+ void receive(FlowFile flowFile, String transitUri, String details, long transmissionMillis);
+
+ /**
+ * Emits a Provenance Event of type
+ * {@link ProvenanceEventType#RECEIVE RECEIVE} that indicates that the given
+ * FlowFile was created from data received from an external source and
+ * provides additional details about the receipt of the FlowFile, such as a
+ * remote system's Distinguished Name.
+ *
+ * @param flowFile the FlowFile that was received
+ * @param transitUri A URI that provides information about the System and
+ * Protocol information over which the transfer occurred. The intent of this
+ * field is such that both the sender and the receiver can publish the
+ * events to an external Enterprise-wide system that is then able to
+ * correlate the SEND and RECEIVE events.
+ * @param sourceSystemFlowFileIdentifier the URI/identifier that the source
+ * system uses to refer to the data; if this value is non-null and is not a
+ * URI, the prefix "urn:tdo:" will be used to form a URI.
+ * @param details details about the receive event; for example, it may be
+ * relevant to include the DN of the sending system
+ * @param transmissionMillis the number of milliseconds taken to transfer
+ * the data
+ */
+ void receive(FlowFile flowFile, String transitUri, String sourceSystemFlowFileIdentifier, String details, long transmissionMillis);
+
+ /**
+ * Emits a Provenance Event of type {@link ProvenanceEventType#SEND SEND}
+ * that indicates that a copy of the given FlowFile was sent to an external
+ * destination. The external destination may be a remote system or may be a
+ * local destination, such as the local file system but is external to NiFi.
+ *
+ * @param flowFile the FlowFile that was sent
+ * @param transitUri A URI that provides information about the System and
+ * Protocol information over which the transfer occurred. The intent of this
+ * field is such that both the sender and the receiver can publish the
+ * events to an external Enterprise-wide system that is then able to
+ * correlate the SEND and RECEIVE events.
+ */
+ void send(FlowFile flowFile, String transitUri);
+
+ /**
+ * Emits a Provenance Event of type {@link ProvenanceEventType#SEND SEND}
+ * that indicates that a copy of the given FlowFile was sent to an external
+ * destination. The external destination may be a remote system or may be a
+ * local destination, such as the local file system but is external to NiFi.
+ *
+ * @param flowFile the FlowFile that was sent
+ * @param transitUri A URI that provides information about the System and
+ * Protocol information over which the transfer occurred. The intent of this
+ * field is such that both the sender and the receiver can publish the
+ * events to an external Enterprise-wide system that is then able to
+ * correlate the SEND and RECEIVE events.
+ * @param details additional details related to the SEND event, such as a
+ * remote system's Distinguished Name
+ */
+ void send(FlowFile flowFile, String transitUri, String details);
+
+ /**
+ * Emits a Provenance Event of type {@link ProvenanceEventType#SEND SEND}
+ * that indicates that a copy of the given FlowFile was sent to an external
+ * destination. The external destination may be a remote system or may be a
+ * local destination, such as the local file system but is external to NiFi.
+ *
+ * @param flowFile the FlowFile that was sent
+ * @param transitUri A URI that provides information about the System and
+ * Protocol information over which the transfer occurred. The intent of this
+ * field is such that both the sender and the receiver can publish the
+ * events to an external Enterprise-wide system that is then able to
+ * correlate the SEND and RECEIVE events.
+ * @param transmissionMillis the number of milliseconds spent sending the
+ * data to the remote system
+ */
+ void send(FlowFile flowFile, String transitUri, long transmissionMillis);
+
+ /**
+ * Emits a Provenance Event of type {@link ProvenanceEventType#SEND SEND}
+ * that indicates that a copy of the given FlowFile was sent to an external
+ * destination. The external destination may be a remote system or may be a
+ * local destination, such as the local file system but is external to NiFi.
+ *
+ * @param flowFile the FlowFile that was sent
+ * @param transitUri A URI that provides information about the System and
+ * Protocol information over which the transfer occurred. The intent of this
+ * field is such that both the sender and the receiver can publish the
+ * events to an external Enterprise-wide system that is then able to
+ * correlate the SEND and RECEIVE events.
+ * @param details additional details related to the SEND event, such as a
+ * remote system's Distinguished Name
+ * @param transmissionMillis the number of milliseconds spent sending the
+ * data to the remote system
+ */
+ void send(FlowFile flowFile, String transitUri, String details, long transmissionMillis);
+
+ /**
+ * Emits a Provenance Event of type {@link ProvenanceEventType#SEND SEND}
+ * that indicates that a copy of the given FlowFile was sent to an external
+ * destination. The external destination may be a remote system or may be a
+ * local destination, such as the local file system but is external to NiFi.
+ *
+ * @param flowFile the FlowFile that was sent
+ * @param transitUri A URI that provides information about the System and
+ * Protocol information over which the transfer occurred. The intent of this
+ * field is such that both the sender and the receiver can publish the
+ * events to an external Enterprise-wide system that is then able to
+ * correlate the SEND and RECEIVE events.
+ * @param force if <code>true</code>, this event will be added to the
+ * Provenance Repository immediately and will still be persisted if the
+ * {@link nifi.processor.ProcessSession ProcessSession} to which this
+ * ProvenanceReporter is associated is rolled back. Otherwise, the Event
+ * will be recorded only on a successful session commit.
+ */
+ void send(FlowFile flowFile, String transitUri, boolean force);
+
+ /**
+ * Emits a Provenance Event of type {@link ProvenanceEventType#SEND SEND}
+ * that indicates that a copy of the given FlowFile was sent to an external
+ * destination. The external destination may be a remote system or may be a
+ * local destination, such as the local file system but is external to NiFi.
+ *
+ * @param flowFile the FlowFile that was sent
+ * @param transitUri A URI that provides information about the System and
+ * Protocol information over which the transfer occurred. The intent of this
+ * field is such that both the sender and the receiver can publish the
+ * events to an external Enterprise-wide system that is then able to
+ * correlate the SEND and RECEIVE events.
+ * @param details additional details related to the SEND event, such as a
+ * remote system's Distinguished Name
+ * @param force if <code>true</code>, this event will be added to the
+ * Provenance Repository immediately and will still be persisted if the
+ * {@link nifi.processor.ProcessSession ProcessSession} to which this
+ * ProvenanceReporter is associated is rolled back. Otherwise, the Event
+ * will be recorded only on a successful session commit.
+ */
+ void send(FlowFile flowFile, String transitUri, String details, boolean force);
+
+ /**
+ * Emits a Provenance Event of type {@link ProvenanceEventType#SEND SEND}
+ * that indicates that a copy of the given FlowFile was sent to an external
+ * destination. The external destination may be a remote system or may be a
+ * local destination, such as the local file system but is external to NiFi.
+ *
+ * @param flowFile the FlowFile that was sent
+ * @param transitUri A URI that provides information about the System and
+ * Protocol information over which the transfer occurred. The intent of this
+ * field is such that both the sender and the receiver can publish the
+ * events to an external Enterprise-wide system that is then able to
+ * correlate the SEND and RECEIVE events.
+ * @param transmissionMillis the number of milliseconds spent sending the
+ * data to the remote system
+ * @param force if <code>true</code>, this event will be added to the
+ * Provenance Repository immediately and will still be persisted if the
+ * {@link nifi.processor.ProcessSession ProcessSession} to which this
+ * ProvenanceReporter is associated is rolled back. Otherwise, the Event
+ * will be recorded only on a successful session commit.
+ */
+ void send(FlowFile flowFile, String transitUri, long transmissionMillis, boolean force);
+
+ /**
+ * Emits a Provenance Event of type {@link ProvenanceEventType#SEND SEND}
+ * that indicates that a copy of the given FlowFile was sent to an external
+ * destination. The external destination may be a remote system or may be a
+ * local destination, such as the local file system but is external to NiFi.
+ *
+ * @param flowFile the FlowFile that was sent
+ * @param transitUri A URI that provides information about the System and
+ * Protocol information over which the transfer occurred. The intent of this
+ * field is such that both the sender and the receiver can publish the
+ * events to an external Enterprise-wide system that is then able to
+ * correlate the SEND and RECEIVE events.
+ * @param details additional details related to the SEND event, such as a
+ * remote system's Distinguished Name
+ * @param transmissionMillis the number of milliseconds spent sending the
+ * data to the remote system
+ * @param force if <code>true</code>, this event will be added to the
+ * Provenance Repository immediately and will still be persisted if the
+ * {@link nifi.processor.ProcessSession ProcessSession} to which this
+ * ProvenanceReporter is associated is rolled back. Otherwise, the Event
+ * will be recorded only on a successful session commit.
+ */
+ void send(FlowFile flowFile, String transitUri, String details, long transmissionMillis, boolean force);
+
+ /**
+ * Emits a Provenance Event of type
+ * {@link ProvenanceEventType#ADDINFO ADDINFO} that provides a linkage
+ * between the given FlowFile and alternate identifier. This information can
+ * be useful if published to an external, enterprise-wide Provenance
+ * tracking system that is able to associate the data between different
+ * processes.
+ *
+ * @param flowFile the FlowFile for which the association should be made
+ * @param alternateIdentifierNamespace the namespace of the alternate system
+ * @param alternateIdentifier the identifier that the alternate system uses
+ * when referring to the data that is encompassed by this FlowFile
+ */
+ void associate(FlowFile flowFile, String alternateIdentifierNamespace, String alternateIdentifier);
+
+ /**
+ * Emits a Provenance Event of type {@link ProvenanceEventType#FORK FORK}
+ * that establishes that the given parent was split into multiple child
+ * FlowFiles. In general, this method does not need to be called by
+ * Processors, as the ProcessSession will handle this automatically for you
+ * when calling {@link ProcessSession#create(FlowFile)}.
+ *
+ * @param parent the FlowFile from which the children are derived
+ * @param children the FlowFiles that are derived from the parent.
+ */
+ void fork(FlowFile parent, Collection<FlowFile> children);
+
+ /**
+ * Emits a Provenance Event of type {@link ProvenanceEventType#FORK FORK}
+ * that establishes that the given parent was split into multiple child
+ * FlowFiles. In general, this method does not need to be called by
+ * Processors, as the ProcessSession will handle this automatically for you
+ * when calling {@link ProcessSession#create(FlowFile)}.
+ *
+ * @param parent the FlowFile from which the children are derived
+ * @param children the FlowFiles that are derived from the parent.
+ * @param details any details pertinent to the fork
+ */
+ void fork(FlowFile parent, Collection<FlowFile> children, String details);
+
+ /**
+ * Emits a Provenance Event of type {@link ProvenanceEventType#FORK FORK}
+ * that establishes that the given parent was split into multiple child
+ * FlowFiles. In general, this method does not need to be called by
+ * Processors, as the ProcessSession will handle this automatically for you
+ * when calling {@link ProcessSession#create(FlowFile)}.
+ *
+ * @param parent the FlowFile from which the children are derived
+ * @param children the FlowFiles that are derived from the parent.
+ * @param forkDuration the number of milliseconds that it took to perform
+ * the task
+ */
+ void fork(FlowFile parent, Collection<FlowFile> children, long forkDuration);
+
+ /**
+ * Emits a Provenance Event of type {@link ProvenanceEventType#FORK FORK}
+ * that establishes that the given parent was split into multiple child
+ * FlowFiles. In general, this method does not need to be called by
+ * Processors, as the ProcessSession will handle this automatically for you
+ * when calling {@link ProcessSession#create(FlowFile)}.
+ *
+ * @param parent the FlowFile from which the children are derived
+ * @param children the FlowFiles that are derived from the parent.
+ * @param details any details pertinent to the fork
+ * @param forkDuration the number of milliseconds that it took to perform
+ * the task
+ */
+ void fork(FlowFile parent, Collection<FlowFile> children, String details, long forkDuration);
+
+ /**
+ * Emits a Provenance Event of type {@link ProvenanceEventType#JOIN JOIN}
+ * that establishes that the given parents were joined together to create a
+ * new child FlowFile. In general, this method does not need to be called by
+ * Processors, as the ProcessSession will handle this automatically for you
+ * when calling {@link ProcessSession#create(FlowFile)}.
+ *
+ * @param parents the FlowFiles that are being joined together to create the
+ * child
+ * @param child the FlowFile that is being created by joining the parents
+ */
+ void join(Collection<FlowFile> parents, FlowFile child);
+
+ /**
+ * Emits a Provenance Event of type {@link ProvenanceEventType#JOIN JOIN}
+ * that establishes that the given parents were joined together to create a
+ * new child FlowFile. In general, this method does not need to be called by
+ * Processors, as the ProcessSession will handle this automatically for you
+ * when calling {@link ProcessSession#create(FlowFile)}.
+ *
+ * @param parents the FlowFiles that are being joined together to create the
+ * child
+ * @param child the FlowFile that is being created by joining the parents
+ * @param details any details pertinent to the event
+ */
+ void join(Collection<FlowFile> parents, FlowFile child, String details);
+
+ /**
+ * Emits a Provenance Event of type {@link ProvenanceEventType#JOIN JOIN}
+ * that establishes that the given parents were joined together to create a
+ * new child FlowFile. In general, this method does not need to be called by
+ * Processors, as the ProcessSession will handle this automatically for you
+ * when calling {@link ProcessSession#create(FlowFile)}.
+ *
+ * @param parents the FlowFiles that are being joined together to create the
+ * child
+ * @param child the FlowFile that is being created by joining the parents
+ * @param joinDuration the number of milliseconds that it took to join the
+ * FlowFiles
+ */
+ void join(Collection<FlowFile> parents, FlowFile child, long joinDuration);
+
+ /**
+ * Emits a Provenance Event of type {@link ProvenanceEventType#JOIN JOIN}
+ * that establishes that the given parents were joined together to create a
+ * new child FlowFile. In general, this method does not need to be called by
+ * Processors, as the ProcessSession will handle this automatically for you
+ * when calling {@link ProcessSession#create(FlowFile)}.
+ *
+ * @param parents the FlowFiles that are being joined together to create the
+ * child
+ * @param child the FlowFile that is being created by joining the parents
+ * @param details any details pertinent to the event
+ * @param joinDuration the number of milliseconds that it took to join the
+ * FlowFiles
+ */
+ void join(Collection<FlowFile> parents, FlowFile child, String details, long joinDuration);
+
+ /**
+ * Emits a Provenance Event of type {@link ProvenanceEventType#CLONE CLONE}
+ * that establishes that the given child is an exact replica of the parent.
+ * In general, this method does not need to be called by Processors, as the
+ * {@link ProcessSession} will handle this automatically for you when
+ * calling {@link ProcessSession#clone(FlowFile)}
+ *
+ * @param parent the FlowFile that was cloned
+ * @param child the clone
+ */
+ void clone(FlowFile parent, FlowFile child);
+
+ /**
+ * Emits a Provenance Event of type
+ * {@link ProvenanceEventType#CONTENT_MODIFIED CONTENT_MODIFIED} that
+ * indicates that the content of the given FlowFile has been modified. One
+ * of the <code>modifyContent</code> methods should be called any time that
+ * the contents of a FlowFile are modified.
+ *
+ * @param flowFile the FlowFile whose content is being modified
+ */
+ void modifyContent(FlowFile flowFile);
+
+ /**
+ * Emits a Provenance Event of type
+ * {@link ProvenanceEventType#CONTENT_MODIFIED CONTENT_MODIFIED} that
+ * indicates that the content of the given FlowFile has been modified. One
+ * of the <code>modifyContent</code> methods should be called any time that
+ * the contents of a FlowFile are modified.
+ *
+ * @param flowFile the FlowFile whose content is being modified
+ * @param details Any details about how the content of the FlowFile has been
+ * modified. Details should not be specified if they can be inferred by
+ * other information in the event, such as the name of the Processor, as
+ * specifying this information will add undue overhead
+ */
+ void modifyContent(FlowFile flowFile, String details);
+
+ /**
+ * Emits a Provenance Event of type
+ * {@link ProvenanceEventType#CONTENT_MODIFIED CONTENT_MODIFIED} that
+ * indicates that the content of the given FlowFile has been modified. One
+ * of the <code>modifyContent</code> methods should be called any time that
+ * the contents of a FlowFile are modified.
+ *
+ * @param flowFile the FlowFile whose content is being modified
+ * @param processingMillis the number of milliseconds spent processing the
+ * FlowFile
+ */
+ void modifyContent(FlowFile flowFile, long processingMillis);
+
+ /**
+ * Emits a Provenance Event of type
+ * {@link ProvenanceEventType#CONTENT_MODIFIED CONTENT_MODIFIED} that
+ * indicates that the content of the given FlowFile has been modified. One
+ * of the <code>modifyContent</code> methods should be called any time that
+ * the contents of a FlowFile are modified.
+ *
+ * @param flowFile the FlowFile whose content is being modified
+ * @param details Any details about how the content of the FlowFile has been
+ * modified. Details should not be specified if they can be inferred by
+ * other information in the event, such as the name of the Processor, as
+ * specifying this information will add undue overhead
+ * @param processingMillis the number of milliseconds spent processing the
+ * FlowFile
+ */
+ void modifyContent(FlowFile flowFile, String details, long processingMillis);
+
+ /**
+ * Emits a Provenance Event of type
+ * {@link ProvenanceEventType#ATTRIBUTES_MODIFIED ATTRIBUTES_MODIFIED} that
+ * indicates that the Attributes of the given FlowFile were updated. It is
+ * not necessary to emit such an event for a FlowFile if other Events are
+ * already emitted by a Processor. For example, one should call both
+ * {@link #modifyContent(FlowFile)} and {@link #modifyAttributes(FlowFile)}
+ * for the same FlowFile in the same Processor. Rather, the Processor should
+ * call just the {@link #modifyContent(FlowFile)}, as the call to
+ * {@link #modifyContent(FlowFile)} will generate a Provenance Event that
+ * already contains all FlowFile attributes. As such, emitting another event
+ * that contains those attributes is unneeded and can result in a
+ * significant amount of overhead for storage and processing.
+ *
+ * @param flowFile the FlowFile whose attributes were modified
+ */
+ void modifyAttributes(FlowFile flowFile);
+
+ /**
+ * Emits a Provenance Event of type
+ * {@link ProvenanceEventType#ATTRIBUTES_MODIFIED ATTRIBUTES_MODIFIED} that
+ * indicates that the Attributes of the given FlowFile were updated. It is
+ * not necessary to emit such an event for a FlowFile if other Events are
+ * already emitted by a Processor. For example, one should call both
+ * {@link #modifyContent(FlowFile)} and {@link #modifyAttributes(FlowFile)}
+ * for the same FlowFile in the same Processor. Rather, the Processor should
+ * call just the {@link #modifyContent(FlowFile)}, as the call to
+ * {@link #modifyContent(FlowFile)} will generate a Provenance Event that
+ * already contains all FlowFile attributes. As such, emitting another event
+ * that contains those attributes is unneeded and can result in a
+ * significant amount of overhead for storage and processing.
+ *
+ * @param flowFile the FlowFile whose attributes were modified
+ * @param details any details should be provided about the attribute
+ * modification
+ */
+ void modifyAttributes(FlowFile flowFile, String details);
+
+ /**
+ * Emits a Provenance Event of type {@link ProvenanceEventType#ROUTE ROUTE}
+ * that indicates that the given FlowFile was routed to the given
+ * {@link Relationship}. <b>Note: </b> this Event is intended for Processors
+ * whose sole job it is to route FlowFiles and should NOT be used as a way
+ * to indicate that the given FlowFile was routed to a standard 'success' or
+ * 'failure' relationship. Doing so can be problematic, as DataFlow Managers
+ * often will loop 'failure' relationships back to the same processor. As
+ * such, emitting a Route event to indicate that a FlowFile was routed to
+ * 'failure' can result in creating thousands of Provenance Events for a
+ * given FlowFile, resulting in a very difficult-to- understand lineage.
+ *
+ * @param flowFile the FlowFile being routed
+ * @param relationship the Relationship to which the FlowFile was routed
+ */
+ void route(FlowFile flowFile, Relationship relationship);
+
+ /**
+ * Emits a Provenance Event of type {@link ProvenanceEventType#ROUTE ROUTE}
+ * that indicates that the given FlowFile was routed to the given
+ * {@link Relationship}. <b>Note: </b> this Event is intended ONLY for
+ * Processors whose sole job it is to route FlowFiles and should NOT be used
+ * as a way to indicate that hte given FlowFile was routed to a standard
+ * 'success' or 'failure' relationship. Doing so can be problematic, as
+ * DataFlow Managers often will loop 'failure' relationships back to the
+ * same processor. As such, emitting a Route event to indicate that a
+ * FlowFile was routed to 'failure' can result in creating thousands of
+ * Provenance Events for a given FlowFile, resulting in a very difficult-to-
+ * understand lineage.
+ *
+ * @param flowFile the FlowFile being routed
+ * @param relationship the Relationship to which the FlowFile was routed
+ * @param details any details pertinent to the Route event, such as why the
+ * FlowFile was routed to the specified Relationship
+ */
+ void route(FlowFile flowFile, Relationship relationship, String details);
+
+ /**
+ * Emits a Provenance Event of type {@link ProvenanceEventType#ROUTE ROUTE}
+ * that indicates that the given FlowFile was routed to the given
+ * {@link Relationship}. <b>Note: </b> this Event is intended ONLY for
+ * Processors whose sole job it is to route FlowFiles and should NOT be used
+ * as a way to indicate that hte given FlowFile was routed to a standard
+ * 'success' or 'failure' relationship. Doing so can be problematic, as
+ * DataFlow Managers often will loop 'failure' relationships back to the
+ * same processor. As such, emitting a Route event to indicate that a
+ * FlowFile was routed to 'failure' can result in creating thousands of
+ * Provenance Events for a given FlowFile, resulting in a very difficult-to-
+ * understand lineage.
+ *
+ * @param flowFile the FlowFile being routed
+ * @param relationship the Relationship to which the FlowFile was routed
+ * @param processingDuration the number of milliseconds that it took to
+ * determine how to route the FlowFile
+ */
+ void route(FlowFile flowFile, Relationship relationship, long processingDuration);
+
+ /**
+ * Emits a Provenance Event of type {@link ProvenanceEventType#ROUTE ROUTE}
+ * that indicates that the given FlowFile was routed to the given
+ * {@link Relationship}. <b>Note: </b> this Event is intended ONLY for
+ * Processors whose sole job it is to route FlowFiles and should NOT be used
+ * as a way to indicate that hte given FlowFile was routed to a standard
+ * 'success' or 'failure' relationship. Doing so can be problematic, as
+ * DataFlow Managers often will loop 'failure' relationships back to the
+ * same processor. As such, emitting a Route event to indicate that a
+ * FlowFile was routed to 'failure' can result in creating thousands of
+ * Provenance Events for a given FlowFile, resulting in a very difficult-to-
+ * understand lineage.
+ *
+ * @param flowFile the FlowFile being routed
+ * @param relationship the Relationship to which the FlowFile was routed
+ * @param details any details pertinent to the Route event, such as why the
+ * FlowFile was routed to the specified Relationship
+ * @param processingDuration the number of milliseconds that it took to
+ * determine how to route the FlowFile
+ */
+ void route(FlowFile flowFile, Relationship relationship, String details, long processingDuration);
+
+ /**
+ * Emits a Provenance Event of type
+ * {@link ProvenanceEventType#CREATE CREATE} that indicates that the given
+ * FlowFile was created by NiFi from data that was not received from an
+ * external entity. If the data was received from an external source, use
+ * the {@link #receive(FlowFile, String)} event instead
+ *
+ * @param flowFile the FlowFile that was created
+ */
+ void create(FlowFile flowFile);
+
+ /**
+ * Emits a Provenance Event of type
+ * {@link ProvenanceEventType#CREATE CREATE} that indicates that the given
+ * FlowFile was created by NiFi from data that was not received from an
+ * external entity. If the data was received from an external source, use
+ * the {@link #receive(FlowFile, String, String, long)} event instead
+ *
+ * @param flowFile the FlowFile that was created
+ * @param details any relevant details about the CREATE event
+ */
+ void create(FlowFile flowFile, String details);
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/provenance/lineage/ComputeLineageResult.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/provenance/lineage/ComputeLineageResult.java b/nifi-api/src/main/java/org/apache/nifi/provenance/lineage/ComputeLineageResult.java
new file mode 100644
index 0000000..e754ff7
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/provenance/lineage/ComputeLineageResult.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance.lineage;
+
+import java.util.Date;
+import java.util.List;
+
+/**
+ *
+ */
+public interface ComputeLineageResult {
+
+ /**
+ * @return all nodes for the graph
+ */
+ public List<LineageNode> getNodes();
+
+ /**
+ * @return all links for the graph
+ */
+ public List<LineageEdge> getEdges();
+
+ /**
+ * @return the date at which this AsynchronousLineageResult will expire
+ */
+ Date getExpiration();
+
+ /**
+ * @return If an error occurred while computing the lineage, this will return the
+ * serialized error; otherwise, returns <code>null</code>
+ */
+ String getError();
+
+ /**
+ * @return an integer between 0 and 100 (inclusive) that indicates what
+ * percentage of completion the computation has reached
+ */
+ int getPercentComplete();
+
+ /**
+ * @return Indicates whether or not the lineage has finished running
+ */
+ boolean isFinished();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/provenance/lineage/ComputeLineageSubmission.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/provenance/lineage/ComputeLineageSubmission.java b/nifi-api/src/main/java/org/apache/nifi/provenance/lineage/ComputeLineageSubmission.java
new file mode 100644
index 0000000..a9df26c
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/provenance/lineage/ComputeLineageSubmission.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance.lineage;
+
+import java.util.Collection;
+import java.util.Date;
+
+public interface ComputeLineageSubmission {
+
+ /**
+ * @return the {@link ComputeLineageResult} that contains the results. The
+ * results may be partial if a call to
+ * {@link ComputeLineageResult#isFinished()} returns <code>false</code>
+ */
+ ComputeLineageResult getResult();
+
+ /**
+ * @return the date at which this lineage was submitted
+ */
+ Date getSubmissionTime();
+
+ /**
+ * @return the generated identifier for this lineage result
+ */
+ String getLineageIdentifier();
+
+ /**
+ * Cancels the lineage computation
+ */
+ void cancel();
+
+ /**
+ * @return <code>true</code> if {@link #cancel()} has been called,
+ * <code>false</code> otherwise
+ */
+ boolean isCanceled();
+
+ /**
+ * @return the type of Lineage Computation that was submitted
+ */
+ LineageComputationType getLineageComputationType();
+
+ /**
+ * @return If the Lineage Computation Type of this submission is
+ * {@link LineageComputationType.EXPAND_CHILDREN} or
+ * {@link LineageComputationType.EXPAND_PARENTS}, indicates the ID event
+ * that is to be expanded; otherwise, returns <code>null</code>
+ */
+ Long getExpandedEventId();
+
+ /**
+ * @return all FlowFile UUID's that are encapsulated in this lineage
+ * computation submission
+ */
+ Collection<String> getLineageFlowFileUuids();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/provenance/lineage/Lineage.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/provenance/lineage/Lineage.java b/nifi-api/src/main/java/org/apache/nifi/provenance/lineage/Lineage.java
new file mode 100644
index 0000000..ff5fee7
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/provenance/lineage/Lineage.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance.lineage;
+
+import java.util.List;
+
+/**
+ * A Data Structure for representing a Directed Graph that depicts the lineage
+ * of a FlowFile and all events that occurred for the FlowFile
+ */
+public interface Lineage {
+
+ /**
+ * @return all nodes for the graph
+ */
+ public List<LineageNode> getNodes();
+
+ /**
+ * @return all links for the graph
+ */
+ public List<LineageEdge> getEdges();
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/provenance/lineage/LineageComputationType.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/provenance/lineage/LineageComputationType.java b/nifi-api/src/main/java/org/apache/nifi/provenance/lineage/LineageComputationType.java
new file mode 100644
index 0000000..be74324
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/provenance/lineage/LineageComputationType.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance.lineage;
+
+/**
+ *
+ */
+public enum LineageComputationType {
+
+ FLOWFILE_LINEAGE,
+ EXPAND_PARENTS,
+ EXPAND_CHILDREN;
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/provenance/lineage/LineageEdge.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/provenance/lineage/LineageEdge.java b/nifi-api/src/main/java/org/apache/nifi/provenance/lineage/LineageEdge.java
new file mode 100644
index 0000000..dba56f3
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/provenance/lineage/LineageEdge.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance.lineage;
+
+public interface LineageEdge {
+
+ String getUuid();
+
+ LineageNode getSource();
+
+ LineageNode getDestination();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/provenance/lineage/LineageNode.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/provenance/lineage/LineageNode.java b/nifi-api/src/main/java/org/apache/nifi/provenance/lineage/LineageNode.java
new file mode 100644
index 0000000..56e865f
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/provenance/lineage/LineageNode.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance.lineage;
+
+public interface LineageNode {
+
+ /**
+ * @return the identifier of the Clustered NiFi Node that generated the
+ * event
+ */
+ @Deprecated
+ String getClusterNodeIdentifier();
+
+ /**
+ * @return the type of the LineageNode
+ */
+ LineageNodeType getNodeType();
+
+ /**
+ * @return the UUID of the FlowFile for which this Node was created
+ */
+ String getFlowFileUuid();
+
+ /**
+ * @return the UUID for this LineageNode
+ */
+ String getIdentifier();
+
+ /**
+ * @return the timestamp that corresponds to this Node. The meaning of the
+ * timestamp may differ between implementations. For example, a
+ * {@link ProvenanceEventLineageNode}'s timestamp indicates the time at
+ * which the event occurred. However, for a Node that reperesents a
+ * FlowFile, for example, the timestamp may represent the time at which the
+ * FlowFile was created
+ */
+ long getTimestamp();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/provenance/lineage/LineageNodeType.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/provenance/lineage/LineageNodeType.java b/nifi-api/src/main/java/org/apache/nifi/provenance/lineage/LineageNodeType.java
new file mode 100644
index 0000000..67e0d61
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/provenance/lineage/LineageNodeType.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance.lineage;
+
+public enum LineageNodeType {
+
+ FLOWFILE_NODE,
+ PROVENANCE_EVENT_NODE;
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/provenance/lineage/ProvenanceEventLineageNode.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/provenance/lineage/ProvenanceEventLineageNode.java b/nifi-api/src/main/java/org/apache/nifi/provenance/lineage/ProvenanceEventLineageNode.java
new file mode 100644
index 0000000..f490496
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/provenance/lineage/ProvenanceEventLineageNode.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance.lineage;
+
+import java.util.List;
+
+import org.apache.nifi.provenance.ProvenanceEventType;
+
+public interface ProvenanceEventLineageNode extends LineageNode {
+
+ ProvenanceEventType getEventType();
+
+ long getEventIdentifier();
+
+ List<String> getParentUuids();
+
+ List<String> getChildUuids();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/provenance/search/Query.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/provenance/search/Query.java b/nifi-api/src/main/java/org/apache/nifi/provenance/search/Query.java
new file mode 100644
index 0000000..3519c14
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/provenance/search/Query.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance.search;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+import java.util.Objects;
+
+
+public class Query {
+
+ private final String identifier;
+ private final List<SearchTerm> searchTerms = new ArrayList<>();
+ private Date startDate;
+ private Date endDate;
+ private String minFileSize;
+ private String maxFileSize;
+ private int maxResults = 1000;
+
+ public Query(final String identifier) {
+ this.identifier = Objects.requireNonNull(identifier);
+ }
+
+ public String getIdentifier() {
+ return identifier;
+ }
+
+ public void addSearchTerm(final SearchTerm searchTerm) {
+ searchTerms.add(searchTerm);
+ }
+
+ public List<SearchTerm> getSearchTerms() {
+ return Collections.unmodifiableList(searchTerms);
+ }
+
+ public Date getStartDate() {
+ return startDate;
+ }
+
+ public void setStartDate(Date startDate) {
+ this.startDate = startDate;
+ }
+
+ public Date getEndDate() {
+ return endDate;
+ }
+
+ public void setEndDate(Date endDate) {
+ this.endDate = endDate;
+ }
+
+ public int getMaxResults() {
+ return maxResults;
+ }
+
+ public void setMaxResults(int maxResults) {
+ this.maxResults = maxResults;
+ }
+
+ public void setMinFileSize(final String fileSize) {
+ this.minFileSize = fileSize;
+ }
+
+ public String getMinFileSize() {
+ return minFileSize;
+ }
+
+ public void setMaxFileSize(final String fileSize) {
+ this.maxFileSize = fileSize;
+ }
+
+ public String getMaxFileSize() {
+ return maxFileSize;
+ }
+
+ @Override
+ public String toString() {
+ return "Query[ " + searchTerms + " ]";
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/provenance/search/QueryResult.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/provenance/search/QueryResult.java b/nifi-api/src/main/java/org/apache/nifi/provenance/search/QueryResult.java
new file mode 100644
index 0000000..0079433
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/provenance/search/QueryResult.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance.search;
+
+import java.util.Date;
+import java.util.List;
+
+import org.apache.nifi.provenance.ProvenanceEventRecord;
+
+public interface QueryResult {
+
+ /**
+ * @return the Provenance events that match the query (up to the limit
+ * specified in the query)
+ */
+ List<ProvenanceEventRecord> getMatchingEvents();
+
+ /**
+ * @return the total number of Provenance Events that hit
+ */
+ long getTotalHitCount();
+
+ /**
+ * @return the number of milliseconds the query took to run
+ */
+ long getQueryTime();
+
+ /**
+ * @return the date at which this QueryResult will expire
+ */
+ Date getExpiration();
+
+ /**
+ * @return If an error occurred while computing the lineage, this will return the
+ * serialized error; otherwise, returns <code>null</code>
+ */
+ String getError();
+
+ /**
+ * @return an integer between 0 and 100 (inclusive) that indicates what
+ * percentage of completion the query has reached
+ */
+ int getPercentComplete();
+
+ /**
+ * @return Indicates whether or not the query has finished running
+ */
+ boolean isFinished();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/provenance/search/QuerySubmission.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/provenance/search/QuerySubmission.java b/nifi-api/src/main/java/org/apache/nifi/provenance/search/QuerySubmission.java
new file mode 100644
index 0000000..4716d2d
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/provenance/search/QuerySubmission.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance.search;
+
+import java.util.Date;
+
+public interface QuerySubmission {
+
+ /**
+ * @return the query that this submission pertains to
+ */
+ Query getQuery();
+
+ /**
+ * @return the {@link QueryResult} for this query. Note that the result is
+ * only a partial result if the result of calling
+ * {@link QueryResult#isFinished()} is <code>false</code>
+ */
+ QueryResult getResult();
+
+ /**
+ * @return the date at which this query was submitted
+ */
+ Date getSubmissionTime();
+
+ /**
+ * @return the generated identifier for this query result
+ */
+ String getQueryIdentifier();
+
+ /**
+ * Cancels the query
+ */
+ void cancel();
+
+ /**
+ * @return <code>true</code> if {@link #cancel()} has been called,
+ * <code>false</code> otherwise
+ */
+ boolean isCanceled();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/provenance/search/SearchTerm.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/provenance/search/SearchTerm.java b/nifi-api/src/main/java/org/apache/nifi/provenance/search/SearchTerm.java
new file mode 100644
index 0000000..573dbd7
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/provenance/search/SearchTerm.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance.search;
+
+public interface SearchTerm {
+
+ SearchableField getSearchableField();
+
+ String getValue();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/provenance/search/SearchTerms.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/provenance/search/SearchTerms.java b/nifi-api/src/main/java/org/apache/nifi/provenance/search/SearchTerms.java
new file mode 100644
index 0000000..aad4d7b
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/provenance/search/SearchTerms.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance.search;
+
+public class SearchTerms {
+
+ public static SearchTerm newSearchTerm(final SearchableField field, final String value) {
+ return new SearchTerm() {
+ @Override
+ public SearchableField getSearchableField() {
+ return field;
+ }
+
+ @Override
+ public String getValue() {
+ return value;
+ }
+
+ @Override
+ public String toString() {
+ return getValue();
+ }
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/provenance/search/SearchableField.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/provenance/search/SearchableField.java b/nifi-api/src/main/java/org/apache/nifi/provenance/search/SearchableField.java
new file mode 100644
index 0000000..85c6154
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/provenance/search/SearchableField.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance.search;
+
+/**
+ * A SearchableField represents a field in a Provenance Event that can be
+ * searched
+ */
+public interface SearchableField {
+
+ /**
+ * @return the identifier that is used to refer to this field
+ */
+ String getIdentifier();
+
+ /**
+ * @return the name of the field that is used when searching the repository
+ */
+ String getSearchableFieldName();
+
+ /**
+ * @return the "friendly" name or "display name" of the field, which may be
+ * more human-readable than the searchable field name
+ */
+ String getFriendlyName();
+
+ /**
+ * @return the type of the data stored in this field
+ */
+ SearchableFieldType getFieldType();
+
+ /**
+ * @return <code>true</code> if this field represents a FlowFile attribute,
+ * <code>false</code> if the field represents a Provenance Event detail,
+ * such as Source System URI
+ */
+ boolean isAttribute();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/provenance/search/SearchableFieldType.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/provenance/search/SearchableFieldType.java b/nifi-api/src/main/java/org/apache/nifi/provenance/search/SearchableFieldType.java
new file mode 100644
index 0000000..51ece58
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/provenance/search/SearchableFieldType.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance.search;
+
+/**
+ *
+ */
+public enum SearchableFieldType {
+
+ STRING,
+ DATE,
+ DATA_SIZE,
+ LONG;
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/remote/RemoteDestination.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/remote/RemoteDestination.java b/nifi-api/src/main/java/org/apache/nifi/remote/RemoteDestination.java
new file mode 100644
index 0000000..c3a34b2
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/remote/RemoteDestination.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.remote;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * A model object for referring to a remote destination (i.e., a Port) for
+ * site-to-site communications
+ */
+public interface RemoteDestination {
+
+ /**
+ * @return the identifier of the remote destination
+ */
+ String getIdentifier();
+
+ /**
+ * @return the human-readable name of the remote destination
+ */
+ String getName();
+
+ /**
+ * @param timeUnit to yield
+ * @return the amount of time that system should pause sending to a
+ * particular node if unable to send data to or receive data from this
+ * endpoint
+ */
+ long getYieldPeriod(TimeUnit timeUnit);
+
+ /**
+ * @return whether or not compression should be used when transferring data
+ * to or receiving data from the remote endpoint
+ */
+ boolean isUseCompression();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/reporting/AbstractReportingTask.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/reporting/AbstractReportingTask.java b/nifi-api/src/main/java/org/apache/nifi/reporting/AbstractReportingTask.java
new file mode 100644
index 0000000..b5afe17
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/reporting/AbstractReportingTask.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.reporting;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.nifi.components.AbstractConfigurableComponent;
+import org.apache.nifi.controller.ControllerServiceLookup;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+
+public abstract class AbstractReportingTask extends AbstractConfigurableComponent implements ReportingTask {
+
+ private String identifier;
+ private String name;
+ private long schedulingNanos;
+ private ControllerServiceLookup serviceLookup;
+ private ComponentLog logger;
+
+ @Override
+ public final void initialize(final ReportingInitializationContext config) throws InitializationException {
+ identifier = config.getIdentifier();
+ logger = config.getLogger();
+ name = config.getName();
+ schedulingNanos = config.getSchedulingPeriod(TimeUnit.NANOSECONDS);
+ serviceLookup = config.getControllerServiceLookup();
+
+ init(config);
+ }
+
+ /**
+ * @return the {@link ControllerServiceLookup} that was passed to the
+ * {@link #init(ProcessorInitializationContext)} method
+ */
+ protected final ControllerServiceLookup getControllerServiceLookup() {
+ return serviceLookup;
+ }
+
+ /**
+ * @return the identifier of this Reporting Task
+ */
+ @Override
+ public String getIdentifier() {
+ return identifier;
+ }
+
+ /**
+ * @return the name of this Reporting Task
+ */
+ protected String getName() {
+ return name;
+ }
+
+ /**
+ * @param timeUnit of scheduling period
+ * @return the amount of times that elapses between the moment that this
+ * ReportingTask finishes its invocation of
+ * {@link #onTrigger(ReportingContext)} and the next time that
+ * {@link #onTrigger(ReportingContext)} is called.
+ */
+ protected long getSchedulingPeriod(final TimeUnit timeUnit) {
+ return timeUnit.convert(schedulingNanos, TimeUnit.NANOSECONDS);
+ }
+
+ /**
+ * Provides a mechanism by which subclasses can perform initialization of
+ * the Reporting Task before it is scheduled to be run
+ *
+ * @param config context
+ * @throws InitializationException if failure to init
+ */
+ protected void init(final ReportingInitializationContext config) throws InitializationException {
+ }
+
+ /**
+ * @return the logger that has been provided to the component by the
+ * framework in its initialize method
+ */
+ protected ComponentLog getLogger() {
+ return logger;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/reporting/Bulletin.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/reporting/Bulletin.java b/nifi-api/src/main/java/org/apache/nifi/reporting/Bulletin.java
new file mode 100644
index 0000000..fe370ae
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/reporting/Bulletin.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.reporting;
+
+import java.util.Date;
+
+/**
+ * A Bulletin is a construct that represents a message that is to be displayed
+ * to users to notify of a specific (usually fleeting) event.
+ */
+public abstract class Bulletin implements Comparable<Bulletin> {
+
+ private final Date timestamp;
+ private final long id;
+ private String nodeAddress;
+ private String level;
+ private String category;
+ private String message;
+
+ private String groupId;
+ private String sourceId;
+ private String sourceName;
+ private ComponentType sourceType;
+
+ protected Bulletin(final long id) {
+ this.timestamp = new Date();
+ this.id = id;
+ }
+
+ public String getNodeAddress() {
+ return nodeAddress;
+ }
+
+ public void setNodeAddress(String nodeAddress) {
+ this.nodeAddress = nodeAddress;
+ }
+
+ public String getCategory() {
+ return category;
+ }
+
+ public void setCategory(String category) {
+ this.category = category;
+ }
+
+ public long getId() {
+ return id;
+ }
+
+ public String getLevel() {
+ return level;
+ }
+
+ public void setLevel(String level) {
+ this.level = level;
+ }
+
+ public String getMessage() {
+ return message;
+ }
+
+ public void setMessage(String message) {
+ this.message = message;
+ }
+
+ public Date getTimestamp() {
+ return timestamp;
+ }
+
+ public String getGroupId() {
+ return groupId;
+ }
+
+ public void setGroupId(String groupId) {
+ this.groupId = groupId;
+ }
+
+ public String getSourceId() {
+ return sourceId;
+ }
+
+ public void setSourceId(String sourceId) {
+ this.sourceId = sourceId;
+ }
+
+ public String getSourceName() {
+ return sourceName;
+ }
+
+ public void setSourceName(String sourceName) {
+ this.sourceName = sourceName;
+ }
+
+ public ComponentType getSourceType() {
+ return sourceType;
+ }
+
+ public void setSourceType(ComponentType sourceType) {
+ this.sourceType = sourceType;
+ }
+
+ @Override
+ public String toString() {
+ return "Bulletin{" + "id=" + id + ", message=" + message + ", sourceName=" + sourceName + ", sourceType=" + sourceType + '}';
+ }
+
+ @Override
+ public int compareTo(Bulletin b) {
+ if (b == null) {
+ return -1;
+ }
+
+ return -Long.compare(getId(), b.getId());
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/reporting/BulletinQuery.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/reporting/BulletinQuery.java b/nifi-api/src/main/java/org/apache/nifi/reporting/BulletinQuery.java
new file mode 100644
index 0000000..7ba2089
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/reporting/BulletinQuery.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.reporting;
+
+import java.util.regex.Pattern;
+
+/**
+ *
+ */
+public class BulletinQuery {
+
+ private final ComponentType sourceType;
+ private final Pattern sourceIdPattern;
+ private final Pattern groupIdPattern;
+ private final Pattern namePattern;
+ private final Pattern messagePattern;
+ private final Long after;
+ private final Integer limit;
+
+ private BulletinQuery(final Builder builder) {
+ this.sourceType = builder.sourceType;
+ this.sourceIdPattern = builder.sourceIdPattern == null ? null : Pattern.compile(builder.sourceIdPattern);
+ this.groupIdPattern = builder.groupIdPattern == null ? null : Pattern.compile(builder.groupIdPattern);
+ this.namePattern = builder.namePattern == null ? null : Pattern.compile(builder.namePattern);
+ this.messagePattern = builder.messagePattern == null ? null : Pattern.compile(builder.messagePattern);
+ this.after = builder.after;
+ this.limit = builder.limit;
+ }
+
+ public ComponentType getSourceType() {
+ return sourceType;
+ }
+
+ public Pattern getSourceIdPattern() {
+ return sourceIdPattern;
+ }
+
+ public Pattern getGroupIdPattern() {
+ return groupIdPattern;
+ }
+
+ public Pattern getNamePattern() {
+ return namePattern;
+ }
+
+ public Pattern getMessagePattern() {
+ return messagePattern;
+ }
+
+ public Long getAfter() {
+ return after;
+ }
+
+ public Integer getLimit() {
+ return limit;
+ }
+
+ public static class Builder {
+
+ private ComponentType sourceType;
+ private String sourceIdPattern;
+ private String groupIdPattern;
+ private String namePattern;
+ private String messagePattern;
+ private Long after;
+ private Integer limit;
+
+ public Builder after(Long after) {
+ this.after = after;
+ return this;
+ }
+
+ public Builder groupIdMatches(String groupId) {
+ this.groupIdPattern = groupId;
+ return this;
+ }
+
+ public Builder sourceIdMatches(String sourceId) {
+ this.sourceIdPattern = sourceId;
+ return this;
+ }
+
+ public Builder sourceType(ComponentType sourceType) {
+ this.sourceType = sourceType;
+ return this;
+ }
+
+ public Builder nameMatches(String name) {
+ this.namePattern = name;
+ return this;
+ }
+
+ public Builder messageMatches(String message) {
+ this.messagePattern = message;
+ return this;
+ }
+
+ public Builder limit(Integer limit) {
+ this.limit = limit;
+ return this;
+ }
+
+ public BulletinQuery build() {
+ return new BulletinQuery(this);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/reporting/BulletinRepository.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/reporting/BulletinRepository.java b/nifi-api/src/main/java/org/apache/nifi/reporting/BulletinRepository.java
new file mode 100644
index 0000000..2679099
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/reporting/BulletinRepository.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.reporting;
+
+import java.util.List;
+
+/**
+ * The BulletinRepository provides a place to store and retrieve
+ * {@link Bulletin}s that have been created by the NiFi Framework and the
+ * Components that are running within the Framework.
+ */
+public interface BulletinRepository {
+
+ /**
+ * Adds a Bulletin to the repository.
+ *
+ * @param bulletin to add
+ */
+ void addBulletin(Bulletin bulletin);
+
+ /**
+ * @return the capacity for the number of bulletins for the controller
+ */
+ int getControllerBulletinCapacity();
+
+ /**
+ * @return the capacity for the number of bulletins per component
+ */
+ int getComponentBulletinCapacity();
+
+ /**
+ * Finds Bulletin's that meet the specified query.
+ *
+ * @param bulletinQuery indicates which bulletins are of interest
+ * @return bulletins that met the query
+ */
+ List<Bulletin> findBulletins(BulletinQuery bulletinQuery);
+
+ /**
+ * Finds all bulletins for the specified group.
+ *
+ * @param groupId id of the group
+ * @return bulletins for the given group
+ */
+ List<Bulletin> findBulletinsForGroupBySource(String groupId);
+
+ /**
+ * Finds all bulletins for the specified group.
+ *
+ * @param groupId id of the group
+ * @param maxPerComponent max responses wanted
+ * @return bulletins found
+ */
+ List<Bulletin> findBulletinsForGroupBySource(String groupId, int maxPerComponent);
+
+ /**
+ * @return all bulletins for the controller
+ */
+ List<Bulletin> findBulletinsForController();
+
+ /**
+ * Finds all bulletins for the controller;
+ *
+ * @param max limits the number of responses
+ * @return all bulletins for the controller
+ */
+ List<Bulletin> findBulletinsForController(int max);
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/reporting/ComponentType.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/reporting/ComponentType.java b/nifi-api/src/main/java/org/apache/nifi/reporting/ComponentType.java
new file mode 100644
index 0000000..97f3538
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/reporting/ComponentType.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.reporting;
+
+/**
+ * An Enumeration for indicating which type of component a Bulletin is associated with
+ */
+public enum ComponentType {
+
+ /**
+ * Bulletin is associated with a Processor
+ */
+ PROCESSOR,
+
+ /**
+ * Bulletin is associated with a Remote Process Group
+ */
+ REMOTE_PROCESS_GROUP,
+
+ /**
+ * Bulletin is associated with an Input Port
+ */
+ INPUT_PORT,
+
+ /**
+ * Bulletin is associated with an Output Port
+ */
+ OUTPUT_PORT,
+
+ /**
+ * Bulletin is associated with a Reporting Task
+ */
+ REPORTING_TASK,
+
+ /**
+ * Bulletin is associated with a Controller Service
+ */
+ CONTROLLER_SERVICE,
+
+ /**
+ * Bulletin is a system-level bulletin, associated with the Flow Controller
+ */
+ FLOW_CONTROLLER;
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/reporting/EventAccess.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/reporting/EventAccess.java b/nifi-api/src/main/java/org/apache/nifi/reporting/EventAccess.java
new file mode 100644
index 0000000..bdc23c2
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/reporting/EventAccess.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.reporting;
+
+import org.apache.nifi.controller.status.ProcessGroupStatus;
+import org.apache.nifi.provenance.ProvenanceEventRecord;
+import org.apache.nifi.provenance.ProvenanceEventRepository;
+
+import java.io.IOException;
+import java.util.List;
+
+public interface EventAccess {
+
+ /**
+ * @return the status for all components in this Controller
+ */
+ ProcessGroupStatus getControllerStatus();
+
+ /**
+ * Convenience method to obtain Provenance Events starting with (and
+ * including) the given ID. If no event exists with that ID, the first event
+ * to be returned will be have an ID greater than <code>firstEventId</code>.
+ *
+ * @param firstEventId the ID of the first event to obtain
+ * @param maxRecords the maximum number of records to obtain
+ * @return event records matching query
+ * @throws java.io.IOException if unable to get records
+ */
+ List<ProvenanceEventRecord> getProvenanceEvents(long firstEventId, final int maxRecords) throws IOException;
+
+ /**
+ * @return the Provenance Event Repository
+ */
+ ProvenanceEventRepository getProvenanceRepository();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-api/src/main/java/org/apache/nifi/reporting/InitializationException.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/reporting/InitializationException.java b/nifi-api/src/main/java/org/apache/nifi/reporting/InitializationException.java
new file mode 100644
index 0000000..c2f21b4
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/reporting/InitializationException.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.reporting;
+
+public class InitializationException extends Exception {
+
+ public InitializationException(String explanation) {
+ super(explanation);
+ }
+
+ public InitializationException(Throwable cause) {
+ super(cause);
+ }
+
+ public InitializationException(String explanation, Throwable cause) {
+ super(explanation, cause);
+ }
+}
[38/51] [partial] nifi git commit: NIFI-850 removed nifi parent,
updated nifi pom, moved all nifi subdirs up one level, fixed readme.
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java
new file mode 100644
index 0000000..07fbac6
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java
@@ -0,0 +1,1313 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language;
+
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionLexer;
+import org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser;
+import org.apache.nifi.attribute.expression.language.evaluation.BooleanEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.DateEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.NumberEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.cast.BooleanCastEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.cast.DateCastEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.cast.NumberCastEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.cast.StringCastEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.AndEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.AppendEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.AttributeEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.ContainsEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.DateToNumberEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.DivideEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.EndsWithEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.EqualsEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.EqualsIgnoreCaseEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.FindEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.FormatEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.GreaterThanEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.GreaterThanOrEqualEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.HostnameEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.IPEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.IndexOfEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.IsEmptyEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.IsNullEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.LastIndexOfEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.LengthEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.LessThanEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.LessThanOrEqualEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.MatchesEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.MinusEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.ModEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.MultiplyEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.NotEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.NotNullEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.NowEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.NumberToDateEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.OneUpSequenceEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.OrEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.PlusEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.PrependEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.ReplaceAllEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.ReplaceEmptyEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.ReplaceEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.ReplaceNullEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.StartsWithEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.StringToDateEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.SubstringAfterEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.SubstringAfterLastEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.SubstringBeforeEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.SubstringBeforeLastEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.SubstringEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.ToLowerEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.ToNumberEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.ToRadixEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.ToStringEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.ToUpperEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.TrimEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.UrlDecodeEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.UrlEncodeEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.UuidEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.literals.BooleanLiteralEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.literals.NumberLiteralEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.literals.StringLiteralEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.literals.ToLiteralEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.reduce.CountEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.reduce.JoinEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.reduce.ReduceEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.selection.AllAttributesEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.selection.AnyAttributeEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.selection.DelineatedAttributeEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.selection.IteratingEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.selection.MultiAttributeEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.selection.MultiMatchAttributeEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.selection.MultiNamedAttributeEvaluator;
+import org.apache.nifi.attribute.expression.language.exception.AttributeExpressionLanguageException;
+import org.apache.nifi.attribute.expression.language.exception.AttributeExpressionLanguageParsingException;
+import org.apache.nifi.expression.AttributeExpression.ResultType;
+import org.apache.nifi.expression.AttributeValueDecorator;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.antlr.runtime.ANTLRStringStream;
+import org.antlr.runtime.CharStream;
+import org.antlr.runtime.CommonTokenStream;
+import org.antlr.runtime.tree.Tree;
+
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ALL_ATTRIBUTES;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ALL_DELINEATED_VALUES;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ALL_MATCHING_ATTRIBUTES;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.AND;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ANY_ATTRIBUTE;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ANY_DELINEATED_VALUE;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ANY_MATCHING_ATTRIBUTE;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.APPEND;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ATTRIBUTE_REFERENCE;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ATTR_NAME;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.CONTAINS;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.COUNT;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.DIVIDE;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ENDS_WITH;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.EQUALS;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.EQUALS_IGNORE_CASE;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.EXPRESSION;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.FALSE;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.FIND;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.FORMAT;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.GREATER_THAN;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.GREATER_THAN_OR_EQUAL;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.HOSTNAME;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.INDEX_OF;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.IP;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.IS_EMPTY;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.IS_NULL;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.JOIN;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.LAST_INDEX_OF;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.LENGTH;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.LESS_THAN;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.LESS_THAN_OR_EQUAL;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.MATCHES;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.MINUS;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.MOD;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.MULTIPLY;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.MULTI_ATTRIBUTE_REFERENCE;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.NEXT_INT;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.NOT;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.NOT_NULL;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.NOW;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.NUMBER;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.OR;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.PLUS;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.PREPEND;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.REPLACE;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.REPLACE_ALL;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.REPLACE_EMPTY;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.REPLACE_NULL;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.STARTS_WITH;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.STRING_LITERAL;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.SUBSTRING;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.SUBSTRING_AFTER;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.SUBSTRING_AFTER_LAST;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.SUBSTRING_BEFORE;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.SUBSTRING_BEFORE_LAST;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TO_DATE;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TO_LITERAL;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TO_LOWER;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TO_NUMBER;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TO_RADIX;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TO_STRING;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TO_UPPER;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TRIM;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TRUE;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.URL_DECODE;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.URL_ENCODE;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.UUID;
+
+import org.apache.nifi.attribute.expression.language.evaluation.selection.MappingEvaluator;
+
+/**
+ * Class used for creating and evaluating NiFi Expression Language. Once a Query
+ * has been created, it may be evaluated using the evaluate methods exactly
+ * once.
+ */
+public class Query {
+
+ private final String query;
+ private final Tree tree;
+ private final Evaluator<?> evaluator;
+ private final AtomicBoolean evaluated = new AtomicBoolean(false);
+
+ private Query(final String query, final Tree tree, final Evaluator<?> evaluator) {
+ this.query = query;
+ this.tree = tree;
+ this.evaluator = evaluator;
+ }
+
+ public static boolean isValidExpression(final String value) {
+ try {
+ validateExpression(value, false);
+ return true;
+ } catch (final AttributeExpressionLanguageParsingException | ProcessException e) {
+ return false;
+ }
+ }
+
+ public static ResultType getResultType(final String value) throws AttributeExpressionLanguageParsingException {
+ return Query.compile(value).getResultType();
+ }
+
+ public static List<ResultType> extractResultTypes(final String value) throws AttributeExpressionLanguageParsingException {
+ final List<ResultType> types = new ArrayList<>();
+
+ for (final Range range : extractExpressionRanges(value)) {
+ final String text = value.substring(range.getStart(), range.getEnd() + 1);
+ types.add(getResultType(text));
+ }
+
+ return types;
+ }
+
+ public static List<String> extractExpressions(final String value) throws AttributeExpressionLanguageParsingException {
+ final List<String> expressions = new ArrayList<>();
+
+ for (final Range range : extractExpressionRanges(value)) {
+ expressions.add(value.substring(range.getStart(), range.getEnd() + 1));
+ }
+
+ return expressions;
+ }
+
+ public static List<Range> extractExpressionRanges(final String value) throws AttributeExpressionLanguageParsingException {
+ final List<Range> ranges = new ArrayList<>();
+ char lastChar = 0;
+ int embeddedCount = 0;
+ int expressionStart = -1;
+ boolean oddDollarCount = false;
+ int backslashCount = 0;
+
+ charLoop:
+ for (int i = 0; i < value.length(); i++) {
+ final char c = value.charAt(i);
+
+ if (expressionStart > -1 && (c == '\'' || c == '"') && (lastChar != '\\' || backslashCount % 2 == 0)) {
+ final int endQuoteIndex = findEndQuoteChar(value, i);
+ if (endQuoteIndex < 0) {
+ break charLoop;
+ }
+
+ i = endQuoteIndex;
+ continue;
+ }
+
+ if (c == '{') {
+ if (oddDollarCount && lastChar == '$') {
+ if (embeddedCount == 0) {
+ expressionStart = i - 1;
+ }
+ }
+
+ // Keep track of the number of opening curly braces that we are embedded within,
+ // if we are within an Expression. If we are outside of an Expression, we can just ignore
+ // curly braces. This allows us to ignore the first character if the value is something
+ // like: { ${abc} }
+ // However, we will count the curly braces if we have something like: ${ $${abc} }
+ if (expressionStart > -1) {
+ embeddedCount++;
+ }
+ } else if (c == '}') {
+ if (embeddedCount <= 0) {
+ continue;
+ }
+
+ if (--embeddedCount == 0) {
+ if (expressionStart > -1) {
+ // ended expression. Add a new range.
+ final Range range = new Range(expressionStart, i);
+ ranges.add(range);
+ }
+
+ expressionStart = -1;
+ }
+ } else if (c == '$') {
+ oddDollarCount = !oddDollarCount;
+ } else if (c == '\\') {
+ backslashCount++;
+ } else {
+ oddDollarCount = false;
+ }
+
+ lastChar = c;
+ }
+
+ return ranges;
+ }
+
+ /**
+ * @param value expression to validate
+ * @param allowSurroundingCharacters whether to allow surrounding chars
+ * @throws AttributeExpressionLanguageParsingException if problems parsing given expression
+ */
+ public static void validateExpression(final String value, final boolean allowSurroundingCharacters) throws AttributeExpressionLanguageParsingException {
+ if (!allowSurroundingCharacters) {
+ final List<Range> ranges = extractExpressionRanges(value);
+ if (ranges.size() > 1) {
+ throw new AttributeExpressionLanguageParsingException("Found multiple Expressions but expected only 1");
+ }
+
+ if (ranges.isEmpty()) {
+ throw new AttributeExpressionLanguageParsingException("No Expressions found");
+ }
+
+ final Range range = ranges.get(0);
+ final String expression = value.substring(range.getStart(), range.getEnd() + 1);
+ Query.compile(expression);
+
+ if (range.getStart() > 0 || range.getEnd() < value.length() - 1) {
+ throw new AttributeExpressionLanguageParsingException("Found characters outside of Expression");
+ }
+ } else {
+ for (final Range range : extractExpressionRanges(value)) {
+ final String expression = value.substring(range.getStart(), range.getEnd() + 1);
+ Query.compile(expression);
+ }
+ }
+ }
+
+ static int findEndQuoteChar(final String value, final int quoteStart) {
+ final char quoteChar = value.charAt(quoteStart);
+
+ int backslashCount = 0;
+ char lastChar = 0;
+ for (int i = quoteStart + 1; i < value.length(); i++) {
+ final char c = value.charAt(i);
+
+ if (c == '\\') {
+ backslashCount++;
+ } else if (c == quoteChar && (backslashCount % 2 == 0 || lastChar != '\\')) {
+ return i;
+ }
+
+ lastChar = c;
+ }
+
+ return -1;
+ }
+
+ static String evaluateExpression(final Tree tree, final String queryText, final Map<String, String> expressionMap, final AttributeValueDecorator decorator) throws ProcessException {
+ final Object evaluated = Query.fromTree(tree, queryText).evaluate(expressionMap).getValue();
+ if (evaluated == null) {
+ return null;
+ }
+
+ final String value = evaluated.toString();
+ final String escaped = value.replace("$$", "$");
+ return decorator == null ? escaped : decorator.decorate(escaped);
+ }
+
+ static String evaluateExpressions(final String rawValue, Map<String, String> expressionMap) throws ProcessException {
+ return evaluateExpressions(rawValue, expressionMap, null);
+ }
+
+ static String evaluateExpressions(final String rawValue) throws ProcessException {
+ return evaluateExpressions(rawValue, createExpressionMap(null), null);
+ }
+
+ static String evaluateExpressions(final String rawValue, final FlowFile flowFile) throws ProcessException {
+ return evaluateExpressions(rawValue, createExpressionMap(flowFile), null);
+ }
+
+ static String evaluateExpressions(final String rawValue, Map<String, String> expressionMap, final AttributeValueDecorator decorator) throws ProcessException {
+ return Query.prepare(rawValue).evaluateExpressions(expressionMap, decorator);
+ }
+
+ public static String evaluateExpressions(final String rawValue, final FlowFile flowFile, final AttributeValueDecorator decorator) throws ProcessException {
+ if (rawValue == null) {
+ return null;
+ }
+
+ final Map<String, String> expressionMap = createExpressionMap(flowFile);
+ return evaluateExpressions(rawValue, expressionMap, decorator);
+ }
+
+ private static Evaluator<?> getRootSubjectEvaluator(final Evaluator<?> evaluator) {
+ if (evaluator == null) {
+ return null;
+ }
+
+ final Evaluator<?> subject = evaluator.getSubjectEvaluator();
+ if (subject == null) {
+ return evaluator;
+ }
+
+ return getRootSubjectEvaluator(subject);
+ }
+
+ /**
+ * Un-escapes ${...} patterns that were escaped
+ *
+ * @param value to un-escape
+ * @return un-escaped value
+ */
+ public static String unescape(final String value) {
+ return value.replaceAll("\\$\\$(?=\\$*\\{.*?\\})", "\\$");
+ }
+
+ static Map<String, String> createExpressionMap(final FlowFile flowFile) {
+ final Map<String, String> attributeMap = flowFile == null ? new HashMap<String, String>() : flowFile.getAttributes();
+ final Map<String, String> envMap = System.getenv();
+ final Map<?, ?> sysProps = System.getProperties();
+
+ final Map<String, String> flowFileProps = new HashMap<>();
+ if (flowFile != null) {
+ flowFileProps.put("flowFileId", String.valueOf(flowFile.getId()));
+ flowFileProps.put("fileSize", String.valueOf(flowFile.getSize()));
+ flowFileProps.put("entryDate", String.valueOf(flowFile.getEntryDate()));
+ flowFileProps.put("lineageStartDate", String.valueOf(flowFile.getLineageStartDate()));
+ }
+
+ return wrap(attributeMap, flowFileProps, envMap, sysProps);
+ }
+
+ private static Map<String, String> wrap(final Map<String, String> attributes, final Map<String, String> flowFileProps,
+ final Map<String, String> env, final Map<?, ?> sysProps) {
+ @SuppressWarnings("rawtypes")
+ final Map[] maps = new Map[]{attributes, flowFileProps, env, sysProps};
+
+ return new Map<String, String>() {
+ @Override
+ public int size() {
+ int size = 0;
+ for (final Map<?, ?> map : maps) {
+ size += map.size();
+ }
+ return size;
+ }
+
+ @Override
+ public boolean isEmpty() {
+ for (final Map<?, ?> map : maps) {
+ if (!map.isEmpty()) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public boolean containsKey(final Object key) {
+ if (key == null) {
+ return false;
+ }
+ if (!(key instanceof String)) {
+ return false;
+ }
+
+ for (final Map<?, ?> map : maps) {
+ if (map.containsKey(key)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public boolean containsValue(final Object value) {
+ for (final Map<?, ?> map : maps) {
+ if (map.containsValue(value)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ @SuppressWarnings("rawtypes")
+ public String get(final Object key) {
+ if (key == null) {
+ throw new IllegalArgumentException("Null Keys are not allowed");
+ }
+ if (!(key instanceof String)) {
+ return null;
+ }
+
+ for (final Map map : maps) {
+ final Object val = map.get(key);
+ if (val != null) {
+ return String.valueOf(val);
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public String put(String key, String value) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public String remove(final Object key) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void putAll(final Map<? extends String, ? extends String> m) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void clear() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ public Set<String> keySet() {
+ final Set<String> keySet = new HashSet<>();
+ for (final Map map : maps) {
+ keySet.addAll(map.keySet());
+ }
+ return keySet;
+ }
+
+ @Override
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ public Collection<String> values() {
+ final Set<String> values = new HashSet<>();
+ for (final Map map : maps) {
+ values.addAll(map.values());
+ }
+ return values;
+ }
+
+ @Override
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ public Set<java.util.Map.Entry<String, String>> entrySet() {
+ final Set<java.util.Map.Entry<String, String>> entrySet = new HashSet<>();
+ for (final Map map : maps) {
+ entrySet.addAll(map.entrySet());
+ }
+ return entrySet;
+ }
+
+ };
+ }
+
+ public static Query fromTree(final Tree tree, final String text) {
+ return new Query(text, tree, buildEvaluator(tree));
+ }
+
+ public static Tree compileTree(final String query) throws AttributeExpressionLanguageParsingException {
+ try {
+ final CommonTokenStream lexerTokenStream = createTokenStream(query);
+ final AttributeExpressionParser parser = new AttributeExpressionParser(lexerTokenStream);
+ final Tree ast = (Tree) parser.query().getTree();
+ final Tree tree = ast.getChild(0);
+
+ // ensure that we are able to build the evaluators, so that we validate syntax
+ final Evaluator<?> evaluator = buildEvaluator(tree);
+ verifyMappingEvaluatorReduced(evaluator);
+ return tree;
+ } catch (final AttributeExpressionLanguageParsingException e) {
+ throw e;
+ } catch (final Exception e) {
+ throw new AttributeExpressionLanguageParsingException(e);
+ }
+ }
+
+ public static PreparedQuery prepare(final String query) throws AttributeExpressionLanguageParsingException {
+ if (query == null) {
+ return new EmptyPreparedQuery(null);
+ }
+
+ final List<Range> ranges = extractExpressionRanges(query);
+
+ if (ranges.isEmpty()) {
+ return new EmptyPreparedQuery(query.replace("$$", "$"));
+ }
+
+ try {
+ final List<String> substrings = new ArrayList<>();
+ final Map<String, Tree> trees = new HashMap<>();
+
+ int lastIndex = 0;
+ for (final Range range : ranges) {
+ if (range.getStart() > lastIndex) {
+ substrings.add(query.substring(lastIndex, range.getStart()).replace("$$", "$"));
+ lastIndex = range.getEnd() + 1;
+ }
+
+ final String treeText = query.substring(range.getStart(), range.getEnd() + 1).replace("$$", "$");
+ substrings.add(treeText);
+ trees.put(treeText, Query.compileTree(treeText));
+ lastIndex = range.getEnd() + 1;
+ }
+
+ final Range lastRange = ranges.get(ranges.size() - 1);
+ if (lastRange.getEnd() + 1 < query.length()) {
+ final String treeText = query.substring(lastRange.getEnd() + 1).replace("$$", "$");
+ substrings.add(treeText);
+ }
+
+ return new StandardPreparedQuery(substrings, trees);
+ } catch (final AttributeExpressionLanguageParsingException e) {
+ return new InvalidPreparedQuery(query, e.getMessage());
+ }
+ }
+
+ public static Query compile(final String query) throws AttributeExpressionLanguageParsingException {
+ try {
+ final CommonTokenStream lexerTokenStream = createTokenStream(query);
+ final AttributeExpressionParser parser = new AttributeExpressionParser(lexerTokenStream);
+ final Tree ast = (Tree) parser.query().getTree();
+ final Tree tree = ast.getChild(0);
+
+ final Evaluator<?> evaluator = buildEvaluator(tree);
+ verifyMappingEvaluatorReduced(evaluator);
+
+ return new Query(query, tree, evaluator);
+ } catch (final AttributeExpressionLanguageParsingException e) {
+ throw e;
+ } catch (final Exception e) {
+ throw new AttributeExpressionLanguageParsingException(e);
+ }
+ }
+
+ private static void verifyMappingEvaluatorReduced(final Evaluator<?> evaluator) {
+ final Evaluator<?> rightMostEvaluator;
+ if (evaluator instanceof IteratingEvaluator) {
+ rightMostEvaluator = ((IteratingEvaluator<?>) evaluator).getLogicEvaluator();
+ } else {
+ rightMostEvaluator = evaluator;
+ }
+
+ Evaluator<?> eval = rightMostEvaluator.getSubjectEvaluator();
+ Evaluator<?> lastEval = rightMostEvaluator;
+ while (eval != null) {
+ if (eval instanceof ReduceEvaluator) {
+ throw new AttributeExpressionLanguageParsingException("Expression attempts to call function '" + lastEval.getToken() + "' on the result of '" + eval.getToken() +
+ "'. This is not allowed. Instead, use \"${literal( ${<embedded expression>} ):" + lastEval.getToken() + "(...)}\"");
+ }
+
+ lastEval = eval;
+ eval = eval.getSubjectEvaluator();
+ }
+
+ // if the result type of the evaluator is BOOLEAN, then it will always
+ // be reduced when evaluator.
+ final ResultType resultType = evaluator.getResultType();
+ if (resultType == ResultType.BOOLEAN) {
+ return;
+ }
+
+ final Evaluator<?> rootEvaluator = getRootSubjectEvaluator(evaluator);
+ if (rootEvaluator != null && rootEvaluator instanceof MultiAttributeEvaluator) {
+ final MultiAttributeEvaluator multiAttrEval = (MultiAttributeEvaluator) rootEvaluator;
+ switch (multiAttrEval.getEvaluationType()) {
+ case ALL_ATTRIBUTES:
+ case ALL_MATCHING_ATTRIBUTES:
+ case ALL_DELINEATED_VALUES: {
+ if (!(evaluator instanceof ReduceEvaluator)) {
+ throw new AttributeExpressionLanguageParsingException("Cannot evaluate expression because it attempts to reference multiple attributes but does not use a reducing function");
+ }
+ break;
+ }
+ default:
+ throw new AttributeExpressionLanguageParsingException("Cannot evaluate expression because it attempts to reference multiple attributes but does not use a reducing function");
+ }
+ }
+ }
+
+ private static CommonTokenStream createTokenStream(final String expression) throws AttributeExpressionLanguageParsingException {
+ final CharStream input = new ANTLRStringStream(expression);
+ final AttributeExpressionLexer lexer = new AttributeExpressionLexer(input);
+ return new CommonTokenStream(lexer);
+ }
+
+ public ResultType getResultType() {
+ return evaluator.getResultType();
+ }
+
+ QueryResult<?> evaluate() {
+ return evaluate(createExpressionMap(null));
+ }
+
+ QueryResult<?> evaluate(final FlowFile flowFile) {
+ return evaluate(createExpressionMap(flowFile));
+ }
+
+ QueryResult<?> evaluate(final Map<String, String> attributes) {
+ if (evaluated.getAndSet(true)) {
+ throw new IllegalStateException("A Query cannot be evaluated more than once");
+ }
+
+ return evaluator.evaluate(attributes);
+ }
+
+ Tree getTree() {
+ return this.tree;
+ }
+
+ @Override
+ public String toString() {
+ return "Query [" + query + "]";
+ }
+
+ private static Evaluator<String> newStringLiteralEvaluator(final String literalValue) {
+ if (literalValue == null || literalValue.length() < 2) {
+ return new StringLiteralEvaluator(literalValue);
+ }
+
+ final List<Range> ranges = extractExpressionRanges(literalValue);
+ if (ranges.isEmpty()) {
+ return new StringLiteralEvaluator(literalValue);
+ }
+
+ final List<Evaluator<?>> evaluators = new ArrayList<>();
+
+ int lastIndex = 0;
+ for (final Range range : ranges) {
+ if (range.getStart() > lastIndex) {
+ evaluators.add(newStringLiteralEvaluator(literalValue.substring(lastIndex, range.getStart())));
+ }
+
+ final String treeText = literalValue.substring(range.getStart(), range.getEnd() + 1);
+ evaluators.add(buildEvaluator(compileTree(treeText)));
+ lastIndex = range.getEnd() + 1;
+ }
+
+ final Range lastRange = ranges.get(ranges.size() - 1);
+ if (lastRange.getEnd() + 1 < literalValue.length()) {
+ final String treeText = literalValue.substring(lastRange.getEnd() + 1);
+ evaluators.add(newStringLiteralEvaluator(treeText));
+ }
+
+ if (evaluators.size() == 1) {
+ return toStringEvaluator(evaluators.get(0));
+ }
+
+ Evaluator<String> lastEvaluator = toStringEvaluator(evaluators.get(0));
+ for (int i = 1; i < evaluators.size(); i++) {
+ lastEvaluator = new AppendEvaluator(lastEvaluator, toStringEvaluator(evaluators.get(i)));
+ }
+
+ return lastEvaluator;
+ }
+
+ private static Evaluator<?> buildEvaluator(final Tree tree) {
+ switch (tree.getType()) {
+ case EXPRESSION: {
+ return buildExpressionEvaluator(tree);
+ }
+ case ATTRIBUTE_REFERENCE: {
+ final Evaluator<?> childEvaluator = buildEvaluator(tree.getChild(0));
+ if (childEvaluator instanceof MultiAttributeEvaluator) {
+ return childEvaluator;
+ }
+ return new AttributeEvaluator(toStringEvaluator(childEvaluator));
+ }
+ case MULTI_ATTRIBUTE_REFERENCE: {
+
+ final Tree functionTypeTree = tree.getChild(0);
+ final int multiAttrType = functionTypeTree.getType();
+ if (multiAttrType == ANY_DELINEATED_VALUE || multiAttrType == ALL_DELINEATED_VALUES) {
+ final Evaluator<String> delineatedValueEvaluator = toStringEvaluator(buildEvaluator(tree.getChild(1)));
+ final Evaluator<String> delimiterEvaluator = toStringEvaluator(buildEvaluator(tree.getChild(2)));
+
+ return new DelineatedAttributeEvaluator(delineatedValueEvaluator, delimiterEvaluator, multiAttrType);
+ }
+
+ final List<String> attributeNames = new ArrayList<>();
+ for (int i = 1; i < tree.getChildCount(); i++) { // skip the first child because that's the name of the multi-attribute function
+ attributeNames.add(newStringLiteralEvaluator(tree.getChild(i).getText()).evaluate(null).getValue());
+ }
+
+ switch (multiAttrType) {
+ case ALL_ATTRIBUTES:
+ for (final String attributeName : attributeNames) {
+ try {
+ FlowFile.KeyValidator.validateKey(attributeName);
+ } catch (final IllegalArgumentException iae) {
+ throw new AttributeExpressionLanguageParsingException("Invalid Attribute Name: " + attributeName + ". " + iae.getMessage());
+ }
+ }
+
+ return new MultiNamedAttributeEvaluator(attributeNames, ALL_ATTRIBUTES);
+ case ALL_MATCHING_ATTRIBUTES:
+ return new MultiMatchAttributeEvaluator(attributeNames, ALL_MATCHING_ATTRIBUTES);
+ case ANY_ATTRIBUTE:
+ for (final String attributeName : attributeNames) {
+ try {
+ FlowFile.KeyValidator.validateKey(attributeName);
+ } catch (final IllegalArgumentException iae) {
+ throw new AttributeExpressionLanguageParsingException("Invalid Attribute Name: " + attributeName + ". " + iae.getMessage());
+ }
+ }
+
+ return new MultiNamedAttributeEvaluator(attributeNames, ANY_ATTRIBUTE);
+ case ANY_MATCHING_ATTRIBUTE:
+ return new MultiMatchAttributeEvaluator(attributeNames, ANY_MATCHING_ATTRIBUTE);
+ default:
+ throw new AssertionError("Illegal Multi-Attribute Reference: " + functionTypeTree.toString());
+ }
+ }
+ case ATTR_NAME: {
+ return newStringLiteralEvaluator(tree.getChild(0).getText());
+ }
+ case NUMBER: {
+ return new NumberLiteralEvaluator(tree.getText());
+ }
+ case STRING_LITERAL: {
+ return newStringLiteralEvaluator(tree.getText());
+ }
+ case TRUE:
+ case FALSE:
+ return buildBooleanEvaluator(tree);
+ case UUID: {
+ return new UuidEvaluator();
+ }
+ case NOW: {
+ return new NowEvaluator();
+ }
+ case TO_LITERAL: {
+ final Evaluator<?> argEvaluator = buildEvaluator(tree.getChild(0));
+ return new ToLiteralEvaluator(argEvaluator);
+ }
+ case IP: {
+ try {
+ return new IPEvaluator();
+ } catch (final UnknownHostException e) {
+ throw new AttributeExpressionLanguageException(e);
+ }
+ }
+ case HOSTNAME: {
+ if (tree.getChildCount() == 0) {
+ try {
+ return new HostnameEvaluator(false);
+ } catch (final UnknownHostException e) {
+ throw new AttributeExpressionLanguageException(e);
+ }
+ } else if (tree.getChildCount() == 1) {
+ final Tree childTree = tree.getChild(0);
+ try {
+ switch (childTree.getType()) {
+ case TRUE:
+ return new HostnameEvaluator(true);
+ case FALSE:
+ return new HostnameEvaluator(false);
+ default:
+ throw new AttributeExpressionLanguageParsingException("Call to hostname() must take 0 or 1 (boolean) parameter");
+ }
+ } catch (final UnknownHostException e) {
+ throw new AttributeExpressionLanguageException(e);
+ }
+ } else {
+ throw new AttributeExpressionLanguageParsingException("Call to hostname() must take 0 or 1 (boolean) parameter");
+ }
+ }
+ case NEXT_INT: {
+ return new OneUpSequenceEvaluator();
+ }
+ default:
+ throw new AttributeExpressionLanguageParsingException("Unexpected token: " + tree.toString());
+ }
+ }
+
+ private static <T> Evaluator<T> addToken(final Evaluator<T> evaluator, final String token) {
+ evaluator.setToken(token);
+ return evaluator;
+ }
+
+ private static Evaluator<Boolean> buildBooleanEvaluator(final Tree tree) {
+ switch (tree.getType()) {
+ case TRUE:
+ return addToken(new BooleanLiteralEvaluator(true), "true");
+ case FALSE:
+ return addToken(new BooleanLiteralEvaluator(false), "true");
+ }
+ throw new AttributeExpressionLanguageParsingException("Cannot build Boolean evaluator from tree " + tree.toString());
+ }
+
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ private static Evaluator<?> buildExpressionEvaluator(final Tree tree) {
+ if (tree.getChildCount() == 0) {
+ throw new AttributeExpressionLanguageParsingException("EXPRESSION tree node has no children");
+ }
+
+ final Evaluator<?> evaluator;
+ if (tree.getChildCount() == 1) {
+ evaluator = buildEvaluator(tree.getChild(0));
+ } else {
+ // we can chain together functions in the form of:
+ // ${x:trim():substring(1,2):trim()}
+ // in this case, the subject of the right-most function is the function to its left; its
+ // subject is the function to its left (the first trim()), and its subject is the value of
+ // the 'x' attribute. We accomplish this logic by iterating over all of the children of the
+ // tree from the right-most child going left-ward.
+ evaluator = buildFunctionExpressionEvaluator(tree, 0);
+ }
+
+ Evaluator<?> chosenEvaluator = evaluator;
+ final Evaluator<?> rootEvaluator = getRootSubjectEvaluator(evaluator);
+ if (rootEvaluator != null) {
+ if (rootEvaluator instanceof MultiAttributeEvaluator) {
+ final MultiAttributeEvaluator multiAttrEval = (MultiAttributeEvaluator) rootEvaluator;
+
+ switch (multiAttrEval.getEvaluationType()) {
+ case ANY_ATTRIBUTE:
+ case ANY_MATCHING_ATTRIBUTE:
+ case ANY_DELINEATED_VALUE:
+ chosenEvaluator = new AnyAttributeEvaluator((BooleanEvaluator) evaluator, multiAttrEval);
+ break;
+ case ALL_ATTRIBUTES:
+ case ALL_MATCHING_ATTRIBUTES:
+ case ALL_DELINEATED_VALUES: {
+ final ResultType resultType = evaluator.getResultType();
+ if (resultType == ResultType.BOOLEAN) {
+ chosenEvaluator = new AllAttributesEvaluator((BooleanEvaluator) evaluator, multiAttrEval);
+ } else if (evaluator instanceof ReduceEvaluator) {
+ chosenEvaluator = new MappingEvaluator((ReduceEvaluator) evaluator, multiAttrEval);
+ } else {
+ throw new AttributeExpressionLanguageException("Cannot evaluate Expression because it attempts to reference multiple attributes but does not use a reducing function");
+ }
+ break;
+ }
+ }
+
+ switch (multiAttrEval.getEvaluationType()) {
+ case ANY_ATTRIBUTE:
+ chosenEvaluator.setToken("anyAttribute");
+ break;
+ case ANY_MATCHING_ATTRIBUTE:
+ chosenEvaluator.setToken("anyMatchingAttribute");
+ break;
+ case ANY_DELINEATED_VALUE:
+ chosenEvaluator.setToken("anyDelineatedValue");
+ break;
+ case ALL_ATTRIBUTES:
+ chosenEvaluator.setToken("allAttributes");
+ break;
+ case ALL_MATCHING_ATTRIBUTES:
+ chosenEvaluator.setToken("allMatchingAttributes");
+ break;
+ case ALL_DELINEATED_VALUES:
+ chosenEvaluator.setToken("allDelineatedValues");
+ break;
+ }
+ }
+ }
+
+ return chosenEvaluator;
+ }
+
+ private static Evaluator<?> buildFunctionExpressionEvaluator(final Tree tree, final int offset) {
+ if (tree.getChildCount() == 0) {
+ throw new AttributeExpressionLanguageParsingException("EXPRESSION tree node has no children");
+ }
+ final int firstChildIndex = tree.getChildCount() - offset - 1;
+ if (firstChildIndex == 0) {
+ return buildEvaluator(tree.getChild(0));
+ }
+
+ final Tree functionTree = tree.getChild(firstChildIndex);
+ final Evaluator<?> subjectEvaluator = buildFunctionExpressionEvaluator(tree, offset + 1);
+
+ final Tree functionNameTree = functionTree.getChild(0);
+ final List<Evaluator<?>> argEvaluators = new ArrayList<>();
+ for (int i = 1; i < functionTree.getChildCount(); i++) {
+ argEvaluators.add(buildEvaluator(functionTree.getChild(i)));
+ }
+ return buildFunctionEvaluator(functionNameTree, subjectEvaluator, argEvaluators);
+ }
+
+ private static List<Evaluator<?>> verifyArgCount(final List<Evaluator<?>> args, final int count, final String functionName) {
+ if (args.size() != count) {
+ throw new AttributeExpressionLanguageParsingException(functionName + "() function takes " + count + " arguments");
+ }
+ return args;
+ }
+
+ private static Evaluator<String> toStringEvaluator(final Evaluator<?> evaluator) {
+ return toStringEvaluator(evaluator, null);
+ }
+
+ private static Evaluator<String> toStringEvaluator(final Evaluator<?> evaluator, final String location) {
+ if (evaluator.getResultType() == ResultType.STRING) {
+ return (StringEvaluator) evaluator;
+ }
+
+ return addToken(new StringCastEvaluator(evaluator), evaluator.getToken());
+ }
+
+ @SuppressWarnings("unchecked")
+ private static Evaluator<Boolean> toBooleanEvaluator(final Evaluator<?> evaluator, final String location) {
+ switch (evaluator.getResultType()) {
+ case BOOLEAN:
+ return (Evaluator<Boolean>) evaluator;
+ case STRING:
+ return addToken(new BooleanCastEvaluator((StringEvaluator) evaluator), evaluator.getToken());
+ default:
+ throw new AttributeExpressionLanguageParsingException("Cannot implicitly convert Data Type " + evaluator.getResultType() + " to " + ResultType.BOOLEAN
+ + (location == null ? "" : " at location [" + location + "]"));
+ }
+
+ }
+
+ private static Evaluator<Boolean> toBooleanEvaluator(final Evaluator<?> evaluator) {
+ return toBooleanEvaluator(evaluator, null);
+ }
+
+ private static Evaluator<Long> toNumberEvaluator(final Evaluator<?> evaluator) {
+ return toNumberEvaluator(evaluator, null);
+ }
+
+ @SuppressWarnings("unchecked")
+ private static Evaluator<Long> toNumberEvaluator(final Evaluator<?> evaluator, final String location) {
+ switch (evaluator.getResultType()) {
+ case NUMBER:
+ return (Evaluator<Long>) evaluator;
+ case STRING:
+ return addToken(new NumberCastEvaluator(evaluator), evaluator.getToken());
+ case DATE:
+ return addToken(new DateToNumberEvaluator((DateEvaluator) evaluator), evaluator.getToken());
+ default:
+ throw new AttributeExpressionLanguageParsingException("Cannot implicitly convert Data Type " + evaluator.getResultType() + " to " + ResultType.NUMBER
+ + (location == null ? "" : " at location [" + location + "]"));
+ }
+ }
+
+ private static DateEvaluator toDateEvaluator(final Evaluator<?> evaluator) {
+ return toDateEvaluator(evaluator, null);
+ }
+
+ private static DateEvaluator toDateEvaluator(final Evaluator<?> evaluator, final String location) {
+ if (evaluator.getResultType() == ResultType.DATE) {
+ return (DateEvaluator) evaluator;
+ }
+
+ return new DateCastEvaluator(evaluator);
+ }
+
+ private static Evaluator<?> buildFunctionEvaluator(final Tree tree, final Evaluator<?> subjectEvaluator, final List<Evaluator<?>> argEvaluators) {
+ switch (tree.getType()) {
+ case TRIM: {
+ verifyArgCount(argEvaluators, 0, "trim");
+ return addToken(new TrimEvaluator(toStringEvaluator(subjectEvaluator)), "trim");
+ }
+ case TO_STRING: {
+ verifyArgCount(argEvaluators, 0, "toString");
+ return addToken(new ToStringEvaluator(subjectEvaluator), "toString");
+ }
+ case TO_LOWER: {
+ verifyArgCount(argEvaluators, 0, "toLower");
+ return addToken(new ToLowerEvaluator(toStringEvaluator(subjectEvaluator)), "toLower");
+ }
+ case TO_UPPER: {
+ verifyArgCount(argEvaluators, 0, "toUpper");
+ return addToken(new ToUpperEvaluator(toStringEvaluator(subjectEvaluator)), "toUpper");
+ }
+ case URL_ENCODE: {
+ verifyArgCount(argEvaluators, 0, "urlEncode");
+ return addToken(new UrlEncodeEvaluator(toStringEvaluator(subjectEvaluator)), "urlEncode");
+ }
+ case URL_DECODE: {
+ verifyArgCount(argEvaluators, 0, "urlDecode");
+ return addToken(new UrlDecodeEvaluator(toStringEvaluator(subjectEvaluator)), "urlDecode");
+ }
+ case SUBSTRING_BEFORE: {
+ verifyArgCount(argEvaluators, 1, "substringBefore");
+ return addToken(new SubstringBeforeEvaluator(toStringEvaluator(subjectEvaluator),
+ toStringEvaluator(argEvaluators.get(0), "first argument to substringBefore")), "substringBefore");
+ }
+ case SUBSTRING_BEFORE_LAST: {
+ verifyArgCount(argEvaluators, 1, "substringBeforeLast");
+ return addToken(new SubstringBeforeLastEvaluator(toStringEvaluator(subjectEvaluator),
+ toStringEvaluator(argEvaluators.get(0), "first argument to substringBeforeLast")), "substringBeforeLast");
+ }
+ case SUBSTRING_AFTER: {
+ verifyArgCount(argEvaluators, 1, "substringAfter");
+ return addToken(new SubstringAfterEvaluator(toStringEvaluator(subjectEvaluator),
+ toStringEvaluator(argEvaluators.get(0), "first argument to substringAfter")), "substringAfter");
+ }
+ case SUBSTRING_AFTER_LAST: {
+ verifyArgCount(argEvaluators, 1, "substringAfterLast");
+ return addToken(new SubstringAfterLastEvaluator(toStringEvaluator(subjectEvaluator),
+ toStringEvaluator(argEvaluators.get(0), "first argument to substringAfterLast")), "substringAfterLast");
+ }
+ case REPLACE_NULL: {
+ verifyArgCount(argEvaluators, 1, "replaceNull");
+ return addToken(new ReplaceNullEvaluator(toStringEvaluator(subjectEvaluator),
+ toStringEvaluator(argEvaluators.get(0), "first argument to replaceNull")), "replaceNull");
+ }
+ case REPLACE_EMPTY: {
+ verifyArgCount(argEvaluators, 1, "replaceEmtpy");
+ return addToken(new ReplaceEmptyEvaluator(toStringEvaluator(subjectEvaluator), toStringEvaluator(argEvaluators.get(0), "first argument to replaceEmpty")), "replaceEmpty");
+ }
+ case REPLACE: {
+ verifyArgCount(argEvaluators, 2, "replace");
+ return addToken(new ReplaceEvaluator(toStringEvaluator(subjectEvaluator),
+ toStringEvaluator(argEvaluators.get(0), "first argument to replace"),
+ toStringEvaluator(argEvaluators.get(1), "second argument to replace")), "replace");
+ }
+ case REPLACE_ALL: {
+ verifyArgCount(argEvaluators, 2, "replaceAll");
+ return addToken(new ReplaceAllEvaluator(toStringEvaluator(subjectEvaluator),
+ toStringEvaluator(argEvaluators.get(0), "first argument to replaceAll"),
+ toStringEvaluator(argEvaluators.get(1), "second argument to replaceAll")), "replaceAll");
+ }
+ case APPEND: {
+ verifyArgCount(argEvaluators, 1, "append");
+ return addToken(new AppendEvaluator(toStringEvaluator(subjectEvaluator),
+ toStringEvaluator(argEvaluators.get(0), "first argument to append")), "append");
+ }
+ case PREPEND: {
+ verifyArgCount(argEvaluators, 1, "prepend");
+ return addToken(new PrependEvaluator(toStringEvaluator(subjectEvaluator),
+ toStringEvaluator(argEvaluators.get(0), "first argument to prepend")), "prepend");
+ }
+ case SUBSTRING: {
+ final int numArgs = argEvaluators.size();
+ if (numArgs == 1) {
+ return addToken(new SubstringEvaluator(toStringEvaluator(subjectEvaluator),
+ toNumberEvaluator(argEvaluators.get(0), "first argument to substring")), "substring");
+ } else if (numArgs == 2) {
+ return addToken(new SubstringEvaluator(toStringEvaluator(subjectEvaluator),
+ toNumberEvaluator(argEvaluators.get(0), "first argument to substring"),
+ toNumberEvaluator(argEvaluators.get(1), "second argument to substring")), "substring");
+ } else {
+ throw new AttributeExpressionLanguageParsingException("substring() function can take either 1 or 2 arguments but cannot take " + numArgs + " arguments");
+ }
+ }
+ case JOIN: {
+ verifyArgCount(argEvaluators, 1, "join");
+ return addToken(new JoinEvaluator(toStringEvaluator(subjectEvaluator), toStringEvaluator(argEvaluators.get(0))), "join");
+ }
+ case COUNT: {
+ verifyArgCount(argEvaluators, 0, "count");
+ return addToken(new CountEvaluator(subjectEvaluator), "count");
+ }
+ case IS_NULL: {
+ verifyArgCount(argEvaluators, 0, "isNull");
+ return addToken(new IsNullEvaluator(toStringEvaluator(subjectEvaluator)), "isNull");
+ }
+ case IS_EMPTY: {
+ verifyArgCount(argEvaluators, 0, "isNull");
+ return addToken(new IsEmptyEvaluator(toStringEvaluator(subjectEvaluator)), "isEmpty");
+ }
+ case NOT_NULL: {
+ verifyArgCount(argEvaluators, 0, "notNull");
+ return addToken(new NotNullEvaluator(toStringEvaluator(subjectEvaluator)), "notNull");
+ }
+ case STARTS_WITH: {
+ verifyArgCount(argEvaluators, 1, "startsWith");
+ return addToken(new StartsWithEvaluator(toStringEvaluator(subjectEvaluator),
+ toStringEvaluator(argEvaluators.get(0), "first argument to startsWith")), "startsWith");
+ }
+ case ENDS_WITH: {
+ verifyArgCount(argEvaluators, 1, "endsWith");
+ return addToken(new EndsWithEvaluator(toStringEvaluator(subjectEvaluator),
+ toStringEvaluator(argEvaluators.get(0), "first argument to endsWith")), "endsWith");
+ }
+ case CONTAINS: {
+ verifyArgCount(argEvaluators, 1, "contains");
+ return addToken(new ContainsEvaluator(toStringEvaluator(subjectEvaluator),
+ toStringEvaluator(argEvaluators.get(0), "first argument to contains")), "contains");
+ }
+ case FIND: {
+ verifyArgCount(argEvaluators, 1, "find");
+ return addToken(new FindEvaluator(toStringEvaluator(subjectEvaluator),
+ toStringEvaluator(argEvaluators.get(0), "first argument to find")), "find");
+ }
+ case MATCHES: {
+ verifyArgCount(argEvaluators, 1, "matches");
+ return addToken(new MatchesEvaluator(toStringEvaluator(subjectEvaluator),
+ toStringEvaluator(argEvaluators.get(0), "first argument to matches")), "matches");
+ }
+ case EQUALS: {
+ verifyArgCount(argEvaluators, 1, "equals");
+ return addToken(new EqualsEvaluator(subjectEvaluator, argEvaluators.get(0)), "equals");
+ }
+ case EQUALS_IGNORE_CASE: {
+ verifyArgCount(argEvaluators, 1, "equalsIgnoreCase");
+ return addToken(new EqualsIgnoreCaseEvaluator(toStringEvaluator(subjectEvaluator),
+ toStringEvaluator(argEvaluators.get(0), "first argument to equalsIgnoreCase")), "equalsIgnoreCase");
+ }
+ case GREATER_THAN: {
+ verifyArgCount(argEvaluators, 1, "gt");
+ return addToken(new GreaterThanEvaluator(toNumberEvaluator(subjectEvaluator),
+ toNumberEvaluator(argEvaluators.get(0), "first argument to gt")), "gt");
+ }
+ case GREATER_THAN_OR_EQUAL: {
+ verifyArgCount(argEvaluators, 1, "ge");
+ return addToken(new GreaterThanOrEqualEvaluator(toNumberEvaluator(subjectEvaluator),
+ toNumberEvaluator(argEvaluators.get(0), "first argument to ge")), "ge");
+ }
+ case LESS_THAN: {
+ verifyArgCount(argEvaluators, 1, "lt");
+ return addToken(new LessThanEvaluator(toNumberEvaluator(subjectEvaluator),
+ toNumberEvaluator(argEvaluators.get(0), "first argument to lt")), "lt");
+ }
+ case LESS_THAN_OR_EQUAL: {
+ verifyArgCount(argEvaluators, 1, "le");
+ return addToken(new LessThanOrEqualEvaluator(toNumberEvaluator(subjectEvaluator),
+ toNumberEvaluator(argEvaluators.get(0), "first argument to le")), "le");
+ }
+ case LENGTH: {
+ verifyArgCount(argEvaluators, 0, "length");
+ return addToken(new LengthEvaluator(toStringEvaluator(subjectEvaluator)), "length");
+ }
+ case TO_DATE: {
+ if (argEvaluators.isEmpty()) {
+ return addToken(new NumberToDateEvaluator(toNumberEvaluator(subjectEvaluator)), "toDate");
+ } else if (subjectEvaluator.getResultType() == ResultType.STRING) {
+ return addToken(new StringToDateEvaluator(toStringEvaluator(subjectEvaluator), toStringEvaluator(argEvaluators.get(0))), "toDate");
+ } else {
+ return addToken(new NumberToDateEvaluator(toNumberEvaluator(subjectEvaluator)), "toDate");
+ }
+ }
+ case TO_NUMBER: {
+ verifyArgCount(argEvaluators, 0, "toNumber");
+ switch (subjectEvaluator.getResultType()) {
+ case STRING:
+ return addToken(new ToNumberEvaluator((StringEvaluator) subjectEvaluator), "toNumber");
+ case DATE:
+ return addToken(new DateToNumberEvaluator((DateEvaluator) subjectEvaluator), "toNumber");
+ default:
+ throw new AttributeExpressionLanguageParsingException(subjectEvaluator + " returns type " + subjectEvaluator.getResultType() + " but expected to get " + ResultType.STRING);
+ }
+ }
+ case TO_RADIX: {
+ if (argEvaluators.size() == 1) {
+ return addToken(new ToRadixEvaluator((NumberEvaluator) subjectEvaluator, toNumberEvaluator(argEvaluators.get(0))), "toRadix");
+ } else {
+ return addToken(new ToRadixEvaluator((NumberEvaluator) subjectEvaluator, toNumberEvaluator(argEvaluators.get(0)), toNumberEvaluator(argEvaluators.get(1))), "toRadix");
+ }
+ }
+ case MOD: {
+ return addToken(new ModEvaluator(toNumberEvaluator(subjectEvaluator), toNumberEvaluator(argEvaluators.get(0))), "mod");
+ }
+ case PLUS: {
+ return addToken(new PlusEvaluator(toNumberEvaluator(subjectEvaluator), toNumberEvaluator(argEvaluators.get(0))), "plus");
+ }
+ case MINUS: {
+ return addToken(new MinusEvaluator(toNumberEvaluator(subjectEvaluator), toNumberEvaluator(argEvaluators.get(0))), "minus");
+ }
+ case MULTIPLY: {
+ return addToken(new MultiplyEvaluator(toNumberEvaluator(subjectEvaluator), toNumberEvaluator(argEvaluators.get(0))), "multiply");
+ }
+ case DIVIDE: {
+ return addToken(new DivideEvaluator(toNumberEvaluator(subjectEvaluator), toNumberEvaluator(argEvaluators.get(0))), "divide");
+ }
+ case INDEX_OF: {
+ verifyArgCount(argEvaluators, 1, "indexOf");
+ return addToken(new IndexOfEvaluator(toStringEvaluator(subjectEvaluator),
+ toStringEvaluator(argEvaluators.get(0), "first argument to indexOf")), "indexOf");
+ }
+ case LAST_INDEX_OF: {
+ verifyArgCount(argEvaluators, 1, "lastIndexOf");
+ return addToken(new LastIndexOfEvaluator(toStringEvaluator(subjectEvaluator),
+ toStringEvaluator(argEvaluators.get(0), "first argument to lastIndexOf")), "lastIndexOf");
+ }
+ case FORMAT: {
+ return addToken(new FormatEvaluator(toDateEvaluator(subjectEvaluator), toStringEvaluator(argEvaluators.get(0), "first argument of format")), "format");
+ }
+ case OR: {
+ return addToken(new OrEvaluator(toBooleanEvaluator(subjectEvaluator), toBooleanEvaluator(argEvaluators.get(0))), "or");
+ }
+ case AND: {
+ return addToken(new AndEvaluator(toBooleanEvaluator(subjectEvaluator), toBooleanEvaluator(argEvaluators.get(0))), "and");
+ }
+ case NOT: {
+ return addToken(new NotEvaluator(toBooleanEvaluator(subjectEvaluator)), "not");
+ }
+ default:
+ throw new AttributeExpressionLanguageParsingException("Expected a Function-type expression but got " + tree.toString());
+ }
+ }
+
+ public static class Range {
+
+ private final int start;
+ private final int end;
+
+ public Range(final int start, final int end) {
+ this.start = start;
+ this.end = end;
+ }
+
+ public int getStart() {
+ return start;
+ }
+
+ public int getEnd() {
+ return end;
+ }
+
+ @Override
+ public String toString() {
+ return start + " - " + end;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StandardAttributeExpression.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StandardAttributeExpression.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StandardAttributeExpression.java
new file mode 100644
index 0000000..49ef6ef
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StandardAttributeExpression.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language;
+
+import org.apache.nifi.expression.AttributeExpression;
+import org.apache.nifi.expression.AttributeValueDecorator;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.exception.ProcessException;
+
+public class StandardAttributeExpression implements AttributeExpression {
+
+ private final Query query;
+
+ public StandardAttributeExpression(final Query query) {
+ this.query = query;
+ }
+
+ @Override
+ public ResultType getResultType() {
+ return query.getResultType();
+ }
+
+ @Override
+ public String evaluate() throws ProcessException {
+ return evaluate((AttributeValueDecorator) null);
+ }
+
+ @Override
+ public String evaluate(final AttributeValueDecorator decorator) throws ProcessException {
+ return evaluate(null, decorator);
+ }
+
+ @Override
+ public String evaluate(final FlowFile flowFile) throws ProcessException {
+ return evaluate(flowFile, null);
+ }
+
+ @Override
+ public String evaluate(final FlowFile flowFile, final AttributeValueDecorator decorator) throws ProcessException {
+ final Object evaluationResult = query.evaluate(flowFile).getValue();
+ if (evaluationResult == null) {
+ return "";
+ }
+
+ String result = evaluationResult.toString();
+ if (decorator != null) {
+ result = decorator.decorate(result);
+ }
+ return Query.unescape(result);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StandardExpressionLanguageCompiler.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StandardExpressionLanguageCompiler.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StandardExpressionLanguageCompiler.java
new file mode 100644
index 0000000..cec73d1
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StandardExpressionLanguageCompiler.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language;
+
+import org.apache.nifi.attribute.expression.language.exception.AttributeExpressionLanguageParsingException;
+import org.apache.nifi.expression.AttributeExpression;
+import org.apache.nifi.expression.ExpressionLanguageCompiler;
+import org.apache.nifi.expression.AttributeExpression.ResultType;
+
+public class StandardExpressionLanguageCompiler implements ExpressionLanguageCompiler {
+
+ @Override
+ public AttributeExpression compile(final String expression) throws IllegalArgumentException {
+ try {
+ return new StandardAttributeExpression(Query.compile(expression));
+ } catch (final AttributeExpressionLanguageParsingException e) {
+ throw new IllegalArgumentException(e.getMessage());
+ }
+ }
+
+ @Override
+ public boolean isValidExpression(final String expression) {
+ return Query.isValidExpression(expression);
+ }
+
+ @Override
+ public String validateExpression(final String expression, final boolean allowSurroundingCharacters) {
+ try {
+ Query.validateExpression(expression, allowSurroundingCharacters);
+ return null;
+ } catch (final AttributeExpressionLanguageParsingException aelpe) {
+ return aelpe.getMessage();
+ }
+ }
+
+ @Override
+ public ResultType getResultType(final String expression) throws IllegalArgumentException {
+ try {
+ return Query.getResultType(expression);
+ } catch (final AttributeExpressionLanguageParsingException e) {
+ throw new IllegalArgumentException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StandardPreparedQuery.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StandardPreparedQuery.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StandardPreparedQuery.java
new file mode 100644
index 0000000..0affb7f
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StandardPreparedQuery.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.nifi.expression.AttributeValueDecorator;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.exception.ProcessException;
+
+import org.antlr.runtime.tree.Tree;
+
+public class StandardPreparedQuery implements PreparedQuery {
+
+ private final List<String> queryStrings;
+ private final Map<String, Tree> trees;
+
+ public StandardPreparedQuery(final List<String> queryStrings, final Map<String, Tree> trees) {
+ this.queryStrings = new ArrayList<>(queryStrings);
+ this.trees = new HashMap<>(trees);
+ }
+
+ @Override
+ public String evaluateExpressions(Map<String, String> attributes) throws ProcessException {
+ return evaluateExpressions(attributes, null);
+ }
+
+ @Override
+ public String evaluateExpressions(final Map<String, String> attributes, final AttributeValueDecorator decorator) throws ProcessException {
+ final StringBuilder sb = new StringBuilder();
+ for (final String val : queryStrings) {
+ final Tree tree = trees.get(val);
+ if (tree == null) {
+ sb.append(val);
+ } else {
+ final String evaluated = Query.evaluateExpression(tree, val, attributes, decorator);
+ if (evaluated != null) {
+ sb.append(evaluated);
+ }
+ }
+ }
+ return sb.toString();
+ }
+
+ @Override
+ public String evaluateExpressions(final FlowFile flowFile, final AttributeValueDecorator decorator) throws ProcessException {
+ final Map<String, String> expressionMap = Query.createExpressionMap(flowFile);
+ return evaluateExpressions(expressionMap, decorator);
+ }
+
+ @Override
+ public String evaluateExpressions() throws ProcessException {
+ return evaluateExpressions((FlowFile) null, null);
+ }
+
+ @Override
+ public String evaluateExpressions(final AttributeValueDecorator decorator) throws ProcessException {
+ return evaluateExpressions((FlowFile) null, decorator);
+ }
+
+ @Override
+ public String evaluateExpressions(final FlowFile flowFile) throws ProcessException {
+ return evaluateExpressions(flowFile, null);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/BooleanEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/BooleanEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/BooleanEvaluator.java
new file mode 100644
index 0000000..907ee95
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/BooleanEvaluator.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation;
+
+import org.apache.nifi.expression.AttributeExpression.ResultType;
+
+public abstract class BooleanEvaluator implements Evaluator<Boolean> {
+ private String token;
+
+ @Override
+ public ResultType getResultType() {
+ return ResultType.BOOLEAN;
+ }
+
+ @Override
+ public int getEvaluationsRemaining() {
+ return 0;
+ }
+
+ @Override
+ public String getToken() {
+ return token;
+ }
+
+ @Override
+ public void setToken(final String token) {
+ this.token = token;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/BooleanQueryResult.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/BooleanQueryResult.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/BooleanQueryResult.java
new file mode 100644
index 0000000..e5ef113
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/BooleanQueryResult.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation;
+
+import org.apache.nifi.expression.AttributeExpression.ResultType;
+
+public class BooleanQueryResult implements QueryResult<Boolean> {
+
+ private final Boolean value;
+
+ public BooleanQueryResult(final Boolean value) {
+ this.value = value;
+ }
+
+ @Override
+ public Boolean getValue() {
+ return value;
+ }
+
+ @Override
+ public ResultType getResultType() {
+ return ResultType.BOOLEAN;
+ }
+
+ @Override
+ public String toString() {
+ return String.valueOf(getValue());
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/DateEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/DateEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/DateEvaluator.java
new file mode 100644
index 0000000..caf3117
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/DateEvaluator.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation;
+
+import java.util.Date;
+
+import org.apache.nifi.expression.AttributeExpression.ResultType;
+
+public abstract class DateEvaluator implements Evaluator<Date> {
+ private String token;
+
+ @Override
+ public ResultType getResultType() {
+ return ResultType.DATE;
+ }
+
+ @Override
+ public int getEvaluationsRemaining() {
+ return 0;
+ }
+
+ @Override
+ public String getToken() {
+ return token;
+ }
+
+ @Override
+ public void setToken(final String token) {
+ this.token = token;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/DateQueryResult.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/DateQueryResult.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/DateQueryResult.java
new file mode 100644
index 0000000..a77bbe9
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/DateQueryResult.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation;
+
+import java.util.Date;
+
+import org.apache.nifi.expression.AttributeExpression.ResultType;
+
+public class DateQueryResult implements QueryResult<Date> {
+
+ private final Date date;
+
+ public DateQueryResult(final Date date) {
+ this.date = date;
+ }
+
+ @Override
+ public Date getValue() {
+ return date;
+ }
+
+ @Override
+ public ResultType getResultType() {
+ return ResultType.DATE;
+ }
+
+ @Override
+ public String toString() {
+ return String.valueOf(getValue());
+ }
+}