You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ma...@apache.org on 2015/11/02 20:32:25 UTC
[01/30] nifi git commit: NIFI-730: Added methods for dropping queued
flowfiles;
refactored swap manager but have not yet started swapping flowfiles in or out
from within the flowfile queue
Repository: nifi
Updated Branches:
refs/heads/master a54962126 -> dbf0c7893
http://git-wip-us.apache.org/repos/asf/nifi/blob/b8c51dc3/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FileSystemSwapManager.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FileSystemSwapManager.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FileSystemSwapManager.java
index c829566..7ab56ed 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FileSystemSwapManager.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FileSystemSwapManager.java
@@ -30,46 +30,29 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Path;
import java.util.ArrayList;
-import java.util.Collection;
+import java.util.Collections;
import java.util.Comparator;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Queue;
import java.util.Set;
-import java.util.SortedMap;
-import java.util.TreeMap;
import java.util.UUID;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
import java.util.regex.Pattern;
-import org.apache.nifi.controller.repository.ConnectionSwapInfo;
+import org.apache.nifi.controller.queue.FlowFileQueue;
+import org.apache.nifi.controller.queue.QueueSize;
import org.apache.nifi.controller.repository.FlowFileRecord;
import org.apache.nifi.controller.repository.FlowFileRepository;
import org.apache.nifi.controller.repository.FlowFileSwapManager;
-import org.apache.nifi.controller.repository.QueueProvider;
import org.apache.nifi.controller.repository.StandardFlowFileRecord;
+import org.apache.nifi.controller.repository.SwapManagerInitializationContext;
import org.apache.nifi.controller.repository.claim.ContentClaim;
import org.apache.nifi.controller.repository.claim.ResourceClaim;
import org.apache.nifi.controller.repository.claim.ResourceClaimManager;
import org.apache.nifi.controller.repository.claim.StandardContentClaim;
-import org.apache.nifi.engine.FlowEngine;
import org.apache.nifi.events.EventReporter;
-import org.apache.nifi.stream.io.BufferedOutputStream;
-import org.apache.nifi.processor.QueueSize;
import org.apache.nifi.reporting.Severity;
-import org.apache.nifi.util.FormatUtils;
+import org.apache.nifi.stream.io.BufferedOutputStream;
import org.apache.nifi.util.NiFiProperties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -85,24 +68,17 @@ public class FileSystemSwapManager implements FlowFileSwapManager {
private static final Pattern SWAP_FILE_PATTERN = Pattern.compile("\\d+-.+\\.swap");
private static final Pattern TEMP_SWAP_FILE_PATTERN = Pattern.compile("\\d+-.+\\.swap\\.part");
- public static final int SWAP_ENCODING_VERSION = 7;
+ public static final int SWAP_ENCODING_VERSION = 8;
public static final String EVENT_CATEGORY = "Swap FlowFiles";
+ private static final Logger logger = LoggerFactory.getLogger(FileSystemSwapManager.class);
- private final ScheduledExecutorService swapQueueIdentifierExecutor;
- private final ScheduledExecutorService swapInExecutor;
- private volatile FlowFileRepository flowFileRepository;
- private volatile EventReporter eventReporter;
-
- // Maintains a mapping of FlowFile Queue to the a QueueLockWrapper, which provides queue locking and necessary state for swapping back in
- private final ConcurrentMap<FlowFileQueue, QueueLockWrapper> swapMap = new ConcurrentHashMap<>();
private final File storageDirectory;
- private final long swapInMillis;
- private final long swapOutMillis;
- private final int swapOutThreadCount;
- private ResourceClaimManager claimManager; // effectively final
+ // effectively final
+ private FlowFileRepository flowFileRepository;
+ private EventReporter eventReporter;
+ private ResourceClaimManager claimManager;
- private static final Logger logger = LoggerFactory.getLogger(FileSystemSwapManager.class);
public FileSystemSwapManager() {
final NiFiProperties properties = NiFiProperties.getInstance();
@@ -112,42 +88,240 @@ public class FileSystemSwapManager implements FlowFileSwapManager {
if (!storageDirectory.exists() && !storageDirectory.mkdirs()) {
throw new RuntimeException("Cannot create Swap Storage directory " + storageDirectory.getAbsolutePath());
}
+ }
- swapQueueIdentifierExecutor = new FlowEngine(1, "Identifies Queues for FlowFile Swapping");
- swapInMillis = FormatUtils.getTimeDuration(properties.getSwapInPeriod(), TimeUnit.MILLISECONDS);
- swapOutMillis = FormatUtils.getTimeDuration(properties.getSwapOutPeriod(), TimeUnit.MILLISECONDS);
- swapOutThreadCount = properties.getSwapOutThreads();
- swapInExecutor = new FlowEngine(properties.getSwapInThreads(), "Swap In FlowFiles");
+ @Override
+ public synchronized void initialize(final SwapManagerInitializationContext initializationContext) {
+ this.claimManager = initializationContext.getResourceClaimManager();
+ this.eventReporter = initializationContext.getEventReporter();
+ this.flowFileRepository = initializationContext.getFlowFileRepository();
}
@Override
+ public String swapOut(final List<FlowFileRecord> toSwap, final FlowFileQueue flowFileQueue) throws IOException {
+ if (toSwap == null || toSwap.isEmpty()) {
+ return null;
+ }
+
+ final File swapFile = new File(storageDirectory, System.currentTimeMillis() + "-" + flowFileQueue.getIdentifier() + "-" + UUID.randomUUID().toString() + ".swap");
+ final File swapTempFile = new File(swapFile.getParentFile(), swapFile.getName() + ".part");
+ final String swapLocation = swapFile.getAbsolutePath();
+
+ try (final FileOutputStream fos = new FileOutputStream(swapTempFile)) {
+ serializeFlowFiles(toSwap, flowFileQueue, swapLocation, fos);
+ fos.getFD().sync();
+ }
+
+ if (swapTempFile.renameTo(swapFile)) {
+ flowFileRepository.swapFlowFilesOut(toSwap, flowFileQueue, swapLocation);
+ } else {
+ error("Failed to swap out FlowFiles from " + flowFileQueue + " due to: Unable to rename swap file from " + swapTempFile + " to " + swapFile);
+ }
+
+ return swapLocation;
+ }
+
+
+ @Override
+ public List<FlowFileRecord> swapIn(final String swapLocation, final FlowFileQueue flowFileQueue) throws IOException {
+ final File swapFile = new File(swapLocation);
+ final List<FlowFileRecord> swappedFlowFiles = peek(swapLocation, flowFileQueue);
+ flowFileRepository.swapFlowFilesIn(swapFile.getAbsolutePath(), swappedFlowFiles, flowFileQueue);
+
+ if (!swapFile.delete()) {
+ warn("Swapped in FlowFiles from file " + swapFile.getAbsolutePath() + " but failed to delete the file; this file should be cleaned up manually");
+ }
+
+ // TODO: When FlowFile Queue performs this operation, it needs to take the following error handling logic into account:
+
+ /*
+ * } catch (final EOFException eof) {
+ * error("Failed to Swap In FlowFiles for " + flowFileQueue + " due to: Corrupt Swap File; will remove this Swap File: " + swapFile);
+ *
+ * if (!swapFile.delete()) {
+ * warn("Failed to remove corrupt Swap File " + swapFile + "; This file should be cleaned up manually");
+ * }
+ * } catch (final FileNotFoundException fnfe) {
+ * error("Failed to Swap In FlowFiles for " + flowFileQueue + " due to: Could not find Swap File " + swapFile);
+ * } catch (final Exception e) {
+ * error("Failed to Swap In FlowFiles for " + flowFileQueue + " due to " + e, e);
+ *
+ * if (swapFile != null) {
+ * queue.add(swapFile);
+ * }
+ * }
+ */
+ return swappedFlowFiles;
+ }
+
+ @Override
+ public List<FlowFileRecord> peek(final String swapLocation, final FlowFileQueue flowFileQueue) throws IOException {
+ final File swapFile = new File(swapLocation);
+ if (!swapFile.exists()) {
+ throw new FileNotFoundException("Failed to swap in FlowFiles from external storage location " + swapLocation + " into FlowFile Queue because the file could not be found");
+ }
+
+ final List<FlowFileRecord> swappedFlowFiles;
+ try (final InputStream fis = new FileInputStream(swapFile);
+ final DataInputStream in = new DataInputStream(fis)) {
+ swappedFlowFiles = deserializeFlowFiles(in, flowFileQueue, swapLocation, claimManager);
+ }
+
+ return swappedFlowFiles;
+ }
+
+
+ @Override
public void purge() {
final File[] swapFiles = storageDirectory.listFiles(new FilenameFilter() {
@Override
public boolean accept(final File dir, final String name) {
- return SWAP_FILE_PATTERN.matcher(name).matches();
+ return SWAP_FILE_PATTERN.matcher(name).matches() || TEMP_SWAP_FILE_PATTERN.matcher(name).matches();
+ }
+ });
+
+ for (final File file : swapFiles) {
+ if (!file.delete()) {
+ warn("Failed to delete Swap File " + file + " when purging FlowFile Swap Manager");
+ }
+ }
+ }
+
+ @Override
+ public List<String> recoverSwapLocations(final FlowFileQueue flowFileQueue) throws IOException {
+ final File[] swapFiles = storageDirectory.listFiles(new FilenameFilter() {
+ @Override
+ public boolean accept(final File dir, final String name) {
+ return SWAP_FILE_PATTERN.matcher(name).matches() || TEMP_SWAP_FILE_PATTERN.matcher(name).matches();
}
});
- if (swapFiles != null) {
- for (final File file : swapFiles) {
- if (!file.delete() && file.exists()) {
- logger.warn("Failed to delete SWAP file {}", file);
+ if (swapFiles == null) {
+ return Collections.emptyList();
+ }
+
+ final List<String> swapLocations = new ArrayList<>();
+ // remove in .part files, as they are partial swap files that did not get written fully.
+ for (final File swapFile : swapFiles) {
+ if (TEMP_SWAP_FILE_PATTERN.matcher(swapFile.getName()).matches()) {
+ if (swapFile.delete()) {
+ logger.info("Removed incomplete/temporary Swap File " + swapFile);
+ } else {
+ warn("Failed to remove incomplete/temporary Swap File " + swapFile + "; this file should be cleaned up manually");
}
+
+ continue;
}
+
+ // split the filename by dashes. The old filenaming scheme was "<timestamp>-<randomuuid>.swap" but the new naming scheme is
+ // "<timestamp>-<queue identifier>-<random uuid>.swap". If we have two dashes, then we can just check if the queue ID is equal
+ // to the id of the queue given and if not we can just move on.
+ final String[] splits = swapFile.getName().split("-");
+ if (splits.length == 3) {
+ final String queueIdentifier = splits[1];
+ if (!queueIdentifier.equals(flowFileQueue.getIdentifier())) {
+ continue;
+ }
+ }
+
+ // Read the queue identifier from the swap file to check if the swap file is for this queue
+ try (final InputStream fis = new FileInputStream(swapFile);
+ final InputStream bufferedIn = new BufferedInputStream(fis);
+ final DataInputStream in = new DataInputStream(bufferedIn)) {
+
+ final int swapEncodingVersion = in.readInt();
+ if (swapEncodingVersion > SWAP_ENCODING_VERSION) {
+ final String errMsg = "Cannot swap FlowFiles in from " + swapFile + " because the encoding version is "
+ + swapEncodingVersion + ", which is too new (expecting " + SWAP_ENCODING_VERSION + " or less)";
+
+ eventReporter.reportEvent(Severity.ERROR, EVENT_CATEGORY, errMsg);
+ throw new IOException(errMsg);
+ }
+
+ final String connectionId = in.readUTF();
+ if (connectionId.equals(flowFileQueue.getIdentifier())) {
+ swapLocations.add(swapFile.getAbsolutePath());
+ }
+ }
+ }
+
+ Collections.sort(swapLocations, new SwapFileComparator());
+ return swapLocations;
+ }
+
+ @Override
+ public QueueSize getSwapSize(final String swapLocation) throws IOException {
+ final File swapFile = new File(swapLocation);
+
+ // read record from disk via the swap file
+ try (final InputStream fis = new FileInputStream(swapFile);
+ final InputStream bufferedIn = new BufferedInputStream(fis);
+ final DataInputStream in = new DataInputStream(bufferedIn)) {
+
+ final int swapEncodingVersion = in.readInt();
+ if (swapEncodingVersion > SWAP_ENCODING_VERSION) {
+ final String errMsg = "Cannot swap FlowFiles in from " + swapFile + " because the encoding version is "
+ + swapEncodingVersion + ", which is too new (expecting " + SWAP_ENCODING_VERSION + " or less)";
+
+ eventReporter.reportEvent(Severity.ERROR, EVENT_CATEGORY, errMsg);
+ throw new IOException(errMsg);
+ }
+
+ in.readUTF(); // ignore Connection ID
+ final int numRecords = in.readInt();
+ final long contentSize = in.readLong();
+
+ return new QueueSize(numRecords, contentSize);
}
}
@Override
- public synchronized void start(final FlowFileRepository flowFileRepository, final QueueProvider connectionProvider, final ResourceClaimManager claimManager, final EventReporter eventReporter) {
- this.claimManager = claimManager;
- this.flowFileRepository = flowFileRepository;
- this.eventReporter = eventReporter;
- swapQueueIdentifierExecutor.scheduleWithFixedDelay(new QueueIdentifier(connectionProvider), swapOutMillis, swapOutMillis, TimeUnit.MILLISECONDS);
- swapInExecutor.scheduleWithFixedDelay(new SwapInTask(), swapInMillis, swapInMillis, TimeUnit.MILLISECONDS);
+ public Long getMaxRecordId(final String swapLocation) throws IOException {
+ final File swapFile = new File(swapLocation);
+
+ // read record from disk via the swap file
+ try (final InputStream fis = new FileInputStream(swapFile);
+ final InputStream bufferedIn = new BufferedInputStream(fis);
+ final DataInputStream in = new DataInputStream(bufferedIn)) {
+
+ final int swapEncodingVersion = in.readInt();
+ if (swapEncodingVersion > SWAP_ENCODING_VERSION) {
+ final String errMsg = "Cannot swap FlowFiles in from " + swapFile + " because the encoding version is "
+ + swapEncodingVersion + ", which is too new (expecting " + SWAP_ENCODING_VERSION + " or less)";
+
+ eventReporter.reportEvent(Severity.ERROR, EVENT_CATEGORY, errMsg);
+ throw new IOException(errMsg);
+ }
+
+ in.readUTF(); // ignore connection id
+ final int numRecords = in.readInt();
+ in.readLong(); // ignore content size
+
+ if (numRecords == 0) {
+ return null;
+ }
+
+ if (swapEncodingVersion > 7) {
+ final long maxRecordId = in.readLong();
+ return maxRecordId;
+ }
+
+ // Before swap encoding version 8, we did not write out the max record id, so we have to read all
+ // swap files to determine the max record id
+ final List<FlowFileRecord> records = deserializeFlowFiles(in, numRecords, swapEncodingVersion, true, claimManager);
+ long maxId = 0L;
+ for (final FlowFileRecord record : records) {
+ if (record.getId() > maxId) {
+ maxId = record.getId();
+ }
+ }
+
+ return maxId;
+ }
}
+
public int serializeFlowFiles(final List<FlowFileRecord> toSwap, final FlowFileQueue queue, final String swapLocation, final OutputStream destination) throws IOException {
if (toSwap == null || toSwap.isEmpty()) {
return 0;
@@ -167,6 +341,16 @@ public class FileSystemSwapManager implements FlowFileSwapManager {
out.writeInt(toSwap.size());
out.writeLong(contentSize);
+ // get the max record id and write that out so that we know it quickly for restoration
+ long maxRecordId = 0L;
+ for (final FlowFileRecord flowFile : toSwap) {
+ if (flowFile.getId() > maxRecordId) {
+ maxRecordId = flowFile.getId();
+ }
+ }
+
+ out.writeLong(maxRecordId);
+
for (final FlowFileRecord flowFile : toSwap) {
out.writeLong(flowFile.getId());
out.writeLong(flowFile.getEntryDate());
@@ -207,7 +391,7 @@ public class FileSystemSwapManager implements FlowFileSwapManager {
out.flush();
}
- logger.info("Successfully swapped out {} FlowFiles from {} to Swap File {}", new Object[]{toSwap.size(), queue, swapLocation});
+ logger.info("Successfully swapped out {} FlowFiles from {} to Swap File {}", new Object[] {toSwap.size(), queue, swapLocation});
return toSwap.size();
}
@@ -231,26 +415,27 @@ public class FileSystemSwapManager implements FlowFileSwapManager {
}
}
- static List<FlowFileRecord> deserializeFlowFiles(final DataInputStream in, final FlowFileQueue queue, final ResourceClaimManager claimManager) throws IOException {
+ static List<FlowFileRecord> deserializeFlowFiles(final DataInputStream in, final FlowFileQueue queue, final String swapLocation, final ResourceClaimManager claimManager) throws IOException {
final int swapEncodingVersion = in.readInt();
if (swapEncodingVersion > SWAP_ENCODING_VERSION) {
throw new IOException("Cannot swap FlowFiles in from SwapFile because the encoding version is "
- + swapEncodingVersion + ", which is too new (expecting " + SWAP_ENCODING_VERSION + " or less)");
+ + swapEncodingVersion + ", which is too new (expecting " + SWAP_ENCODING_VERSION + " or less)");
}
final String connectionId = in.readUTF();
if (!connectionId.equals(queue.getIdentifier())) {
- throw new IllegalArgumentException("Cannot restore Swap File because the file indicates that records belong to Connection with ID " + connectionId + " but received Connection " + queue);
+ throw new IllegalArgumentException("Cannot restore contents from FlowFile Swap File " + swapLocation +
+ " because the file indicates that records belong to Connection with ID " + connectionId + " but attempted to swap those records into " + queue);
}
final int numRecords = in.readInt();
- in.readLong(); // Content Size
+ in.readLong(); // Content Size
- return deserializeFlowFiles(in, numRecords, queue, swapEncodingVersion, false, claimManager);
+ return deserializeFlowFiles(in, numRecords, swapEncodingVersion, false, claimManager);
}
- static List<FlowFileRecord> deserializeFlowFiles(final DataInputStream in, final int numFlowFiles, final FlowFileQueue queue,
- final int serializationVersion, final boolean incrementContentClaims, final ResourceClaimManager claimManager) throws IOException {
+ static List<FlowFileRecord> deserializeFlowFiles(final DataInputStream in, final int numFlowFiles,
+ final int serializationVersion, final boolean incrementContentClaims, final ResourceClaimManager claimManager) throws IOException {
final List<FlowFileRecord> flowFiles = new ArrayList<>();
for (int i = 0; i < numFlowFiles; i++) {
// legacy encoding had an "action" because it used to be couple with FlowFile Repository code
@@ -395,109 +580,6 @@ public class FileSystemSwapManager implements FlowFileSwapManager {
}
}
- private class QueueIdentifier implements Runnable {
-
- private final QueueProvider connectionProvider;
-
- public QueueIdentifier(final QueueProvider connectionProvider) {
- this.connectionProvider = connectionProvider;
- }
-
- @Override
- public void run() {
- final Collection<FlowFileQueue> allQueues = connectionProvider.getAllQueues();
- final BlockingQueue<FlowFileQueue> connectionQueue = new LinkedBlockingQueue<>(allQueues);
-
- final ThreadFactory threadFactory = new ThreadFactory() {
- @Override
- public Thread newThread(final Runnable r) {
- final Thread t = new Thread(r);
- t.setName("Swap Out FlowFiles");
- return t;
- }
- };
-
- final ExecutorService workerExecutor = Executors.newFixedThreadPool(swapOutThreadCount, threadFactory);
- for (int i = 0; i < swapOutThreadCount; i++) {
- workerExecutor.submit(new SwapOutTask(connectionQueue));
- }
-
- workerExecutor.shutdown();
-
- try {
- workerExecutor.awaitTermination(10, TimeUnit.MINUTES);
- } catch (final InterruptedException e) {
- // oh well...
- }
- }
- }
-
- private class SwapInTask implements Runnable {
-
- @Override
- public void run() {
- for (final Map.Entry<FlowFileQueue, QueueLockWrapper> entry : swapMap.entrySet()) {
- final FlowFileQueue flowFileQueue = entry.getKey();
-
- // if queue is more than 60% of its swap threshold, don't swap flowfiles in
- if (flowFileQueue.unswappedSize() >= flowFileQueue.getSwapThreshold() * 0.6F) {
- continue;
- }
-
- final QueueLockWrapper queueLockWrapper = entry.getValue();
- if (queueLockWrapper.getLock().tryLock()) {
- try {
- final Queue<File> queue = queueLockWrapper.getQueue();
-
- // Swap FlowFiles in until we hit 90% of the threshold, or until we're out of files.
- while (flowFileQueue.unswappedSize() < flowFileQueue.getSwapThreshold() * 0.9F) {
- File swapFile = null;
- try {
- swapFile = queue.poll();
- if (swapFile == null) {
- break;
- }
-
- try (final InputStream fis = new FileInputStream(swapFile);
- final DataInputStream in = new DataInputStream(fis)) {
- final List<FlowFileRecord> swappedFlowFiles = deserializeFlowFiles(in, flowFileQueue, claimManager);
- flowFileRepository.swapFlowFilesIn(swapFile.getAbsolutePath(), swappedFlowFiles, flowFileQueue);
- flowFileQueue.putSwappedRecords(swappedFlowFiles);
- }
-
- if (!swapFile.delete()) {
- warn("Swapped in FlowFiles from file " + swapFile.getAbsolutePath() + " but failed to delete the file; this file should be cleaned up manually");
- }
- } catch (final EOFException eof) {
- error("Failed to Swap In FlowFiles for " + flowFileQueue + " due to: Corrupt Swap File; will remove this Swap File: " + swapFile);
-
- if (!swapFile.delete()) {
- warn("Failed to remove corrupt Swap File " + swapFile + "; This file should be cleaned up manually");
- }
- } catch (final FileNotFoundException fnfe) {
- error("Failed to Swap In FlowFiles for " + flowFileQueue + " due to: Could not find Swap File " + swapFile);
- } catch (final Exception e) {
- error("Failed to Swap In FlowFiles for " + flowFileQueue + " due to " + e, e);
-
- if (swapFile != null) {
- queue.add(swapFile);
- }
- }
- }
- } finally {
- queueLockWrapper.getLock().unlock();
- }
- }
- }
- }
- }
-
- private void error(final String error, final Throwable t) {
- error(error);
- if (logger.isDebugEnabled()) {
- logger.error("", t);
- }
- }
private void error(final String error) {
logger.error(error);
@@ -513,199 +595,9 @@ public class FileSystemSwapManager implements FlowFileSwapManager {
}
}
- private class SwapOutTask implements Runnable {
-
- private final BlockingQueue<FlowFileQueue> connectionQueue;
-
- public SwapOutTask(final BlockingQueue<FlowFileQueue> connectionQueue) {
- this.connectionQueue = connectionQueue;
- }
-
- @Override
- public void run() {
- while (true) {
- final FlowFileQueue flowFileQueue = connectionQueue.poll();
- if (flowFileQueue == null) {
- logger.debug("No more FlowFile Queues to Swap Out");
- return;
- }
-
- if (logger.isDebugEnabled()) {
- logger.debug("{} has {} FlowFiles to swap out", flowFileQueue, flowFileQueue.getSwapQueueSize());
- }
-
- while (flowFileQueue.getSwapQueueSize() >= MINIMUM_SWAP_COUNT) {
- final File swapFile = new File(storageDirectory, System.currentTimeMillis() + "-" + UUID.randomUUID().toString() + ".swap");
- final File swapTempFile = new File(swapFile.getParentFile(), swapFile.getName() + ".part");
- final String swapLocation = swapFile.getAbsolutePath();
- final List<FlowFileRecord> toSwap = flowFileQueue.pollSwappableRecords();
-
- int recordsSwapped;
- try {
- try (final FileOutputStream fos = new FileOutputStream(swapTempFile)) {
- recordsSwapped = serializeFlowFiles(toSwap, flowFileQueue, swapLocation, fos);
- fos.getFD().sync();
- }
-
- if (swapTempFile.renameTo(swapFile)) {
- flowFileRepository.swapFlowFilesOut(toSwap, flowFileQueue, swapLocation);
- } else {
- error("Failed to swap out FlowFiles from " + flowFileQueue + " due to: Unable to rename swap file from " + swapTempFile + " to " + swapFile);
- recordsSwapped = 0;
- }
- } catch (final IOException ioe) {
- recordsSwapped = 0;
- flowFileQueue.putSwappedRecords(toSwap);
- error("Failed to swap out " + toSwap.size() + " FlowFiles from " + flowFileQueue + " to Swap File " + swapLocation + " due to " + ioe, ioe);
- }
-
- if (recordsSwapped > 0) {
- QueueLockWrapper swapQueue = swapMap.get(flowFileQueue);
- if (swapQueue == null) {
- swapQueue = new QueueLockWrapper(new LinkedBlockingQueue<File>());
- final QueueLockWrapper oldQueue = swapMap.putIfAbsent(flowFileQueue, swapQueue);
- if (oldQueue != null) {
- swapQueue = oldQueue;
- }
- }
-
- swapQueue.getQueue().add(swapFile);
- } else {
- swapTempFile.delete();
- }
- }
- }
- }
- }
-
- /**
- * Recovers FlowFiles from all Swap Files, returning the largest FlowFile ID that was recovered.
- *
- * @param queueProvider provider
- * @return the largest FlowFile ID that was recovered
- */
- @Override
- public long recoverSwappedFlowFiles(final QueueProvider queueProvider, final ResourceClaimManager claimManager) {
- final File[] swapFiles = storageDirectory.listFiles(new FilenameFilter() {
- @Override
- public boolean accept(final File dir, final String name) {
- return SWAP_FILE_PATTERN.matcher(name).matches() || TEMP_SWAP_FILE_PATTERN.matcher(name).matches();
- }
- });
-
- if (swapFiles == null) {
- return 0L;
- }
-
- final Collection<FlowFileQueue> allQueues = queueProvider.getAllQueues();
- final Map<String, FlowFileQueue> queueMap = new HashMap<>();
- for (final FlowFileQueue queue : allQueues) {
- queueMap.put(queue.getIdentifier(), queue);
- }
-
- final ConnectionSwapInfo swapInfo = new ConnectionSwapInfo();
- int swappedCount = 0;
- long swappedBytes = 0L;
- long maxRecoveredId = 0L;
-
- for (final File swapFile : swapFiles) {
- if (TEMP_SWAP_FILE_PATTERN.matcher(swapFile.getName()).matches()) {
- if (swapFile.delete()) {
- logger.info("Removed incomplete/temporary Swap File " + swapFile);
- } else {
- warn("Failed to remove incomplete/temporary Swap File " + swapFile + "; this file should be cleaned up manually");
- }
-
- continue;
- }
-
- // read record to disk via the swap file
- try (final InputStream fis = new FileInputStream(swapFile);
- final InputStream bufferedIn = new BufferedInputStream(fis);
- final DataInputStream in = new DataInputStream(bufferedIn)) {
-
- final int swapEncodingVersion = in.readInt();
- if (swapEncodingVersion > SWAP_ENCODING_VERSION) {
- final String errMsg = "Cannot swap FlowFiles in from " + swapFile + " because the encoding version is "
- + swapEncodingVersion + ", which is too new (expecting " + SWAP_ENCODING_VERSION + " or less)";
-
- eventReporter.reportEvent(Severity.ERROR, EVENT_CATEGORY, errMsg);
- throw new IOException(errMsg);
- }
-
- final String connectionId = in.readUTF();
- final FlowFileQueue queue = queueMap.get(connectionId);
- if (queue == null) {
- error("Cannot recover Swapped FlowFiles from Swap File " + swapFile + " because the FlowFiles belong to a Connection with ID "
- + connectionId + " and that Connection does not exist");
- continue;
- }
-
- final int numRecords = in.readInt();
- final long contentSize = in.readLong();
-
- swapInfo.addSwapSizeInfo(connectionId, swapFile.getAbsolutePath(), new QueueSize(numRecords, contentSize));
- swappedCount += numRecords;
- swappedBytes += contentSize;
-
- final List<FlowFileRecord> records = deserializeFlowFiles(in, numRecords, queue, swapEncodingVersion, true, claimManager);
- long maxId = 0L;
- for (final FlowFileRecord record : records) {
- if (record.getId() > maxId) {
- maxId = record.getId();
- }
- }
-
- if (maxId > maxRecoveredId) {
- maxRecoveredId = maxId;
- }
- } catch (final IOException ioe) {
- error("Cannot recover Swapped FlowFiles from Swap File " + swapFile + " due to " + ioe, ioe);
- }
- }
-
- restoreSwapLocations(queueMap.values(), swapInfo);
- logger.info("Recovered {} FlowFiles ({} bytes) from Swap Files", swappedCount, swappedBytes);
- return maxRecoveredId;
- }
-
- public void restoreSwapLocations(final Collection<FlowFileQueue> flowFileQueues, final ConnectionSwapInfo swapInfo) {
- for (final FlowFileQueue queue : flowFileQueues) {
- final String queueId = queue.getIdentifier();
- final Collection<String> swapFileLocations = swapInfo.getSwapFileLocations(queueId);
- if (swapFileLocations == null || swapFileLocations.isEmpty()) {
- continue;
- }
-
- final SortedMap<String, QueueSize> sortedFileQueueMap = new TreeMap<>(new SwapFileComparator());
- for (final String swapFileLocation : swapFileLocations) {
- final QueueSize queueSize = swapInfo.getSwappedSize(queueId, swapFileLocation);
- sortedFileQueueMap.put(swapFileLocation, queueSize);
- }
-
- QueueLockWrapper fileQueue = swapMap.get(queue);
- if (fileQueue == null) {
- fileQueue = new QueueLockWrapper(new LinkedBlockingQueue<File>());
- swapMap.put(queue, fileQueue);
- }
-
- for (final Map.Entry<String, QueueSize> innerEntry : sortedFileQueueMap.entrySet()) {
- final File swapFile = new File(innerEntry.getKey());
- final QueueSize size = innerEntry.getValue();
- fileQueue.getQueue().add(swapFile);
- queue.incrementSwapCount(size.getObjectCount(), size.getByteCount());
- }
- }
- }
- @Override
- public void shutdown() {
- swapQueueIdentifierExecutor.shutdownNow();
- swapInExecutor.shutdownNow();
- }
private static class SwapFileComparator implements Comparator<String> {
-
@Override
public int compare(final String o1, final String o2) {
if (o1 == o2) {
@@ -755,34 +647,4 @@ public class FileSystemSwapManager implements FlowFileSwapManager {
}
}
- private static class QueueLockWrapper {
-
- private final Lock lock = new ReentrantLock();
- private final Queue<File> queue;
-
- public QueueLockWrapper(final Queue<File> queue) {
- this.queue = queue;
- }
-
- public Queue<File> getQueue() {
- return queue;
- }
-
- public Lock getLock() {
- return lock;
- }
-
- @Override
- public int hashCode() {
- return queue.hashCode();
- }
-
- @Override
- public boolean equals(final Object obj) {
- if (obj instanceof QueueLockWrapper) {
- return queue.equals(((QueueLockWrapper) obj).queue);
- }
- return false;
- }
- }
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b8c51dc3/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java
index d9c3f39..23746ce 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java
@@ -80,6 +80,8 @@ import org.apache.nifi.controller.exception.ComponentLifeCycleException;
import org.apache.nifi.controller.exception.ProcessorInstantiationException;
import org.apache.nifi.controller.label.Label;
import org.apache.nifi.controller.label.StandardLabel;
+import org.apache.nifi.controller.queue.FlowFileQueue;
+import org.apache.nifi.controller.queue.QueueSize;
import org.apache.nifi.controller.reporting.ReportingTaskInstantiationException;
import org.apache.nifi.controller.reporting.ReportingTaskProvider;
import org.apache.nifi.controller.reporting.StandardReportingInitializationContext;
@@ -97,6 +99,7 @@ import org.apache.nifi.controller.repository.RepositoryStatusReport;
import org.apache.nifi.controller.repository.StandardCounterRepository;
import org.apache.nifi.controller.repository.StandardFlowFileRecord;
import org.apache.nifi.controller.repository.StandardRepositoryRecord;
+import org.apache.nifi.controller.repository.SwapManagerInitializationContext;
import org.apache.nifi.controller.repository.claim.ContentClaim;
import org.apache.nifi.controller.repository.claim.ContentDirection;
import org.apache.nifi.controller.repository.claim.ResourceClaim;
@@ -152,7 +155,6 @@ import org.apache.nifi.nar.NarCloseable;
import org.apache.nifi.nar.NarThreadContextClassLoader;
import org.apache.nifi.processor.Processor;
import org.apache.nifi.processor.ProcessorInitializationContext;
-import org.apache.nifi.processor.QueueSize;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.SimpleProcessLogger;
import org.apache.nifi.processor.StandardProcessorInitializationContext;
@@ -216,7 +218,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
public static final String SCHEDULE_MINIMUM_NANOSECONDS = "flowcontroller.minimum.nanoseconds";
public static final String GRACEFUL_SHUTDOWN_PERIOD = "nifi.flowcontroller.graceful.shutdown.seconds";
public static final long DEFAULT_GRACEFUL_SHUTDOWN_SECONDS = 10;
- public static final int METRICS_RESERVOIR_SIZE = 288; // 1 day worth of 5-minute captures
+ public static final int METRICS_RESERVOIR_SIZE = 288; // 1 day worth of 5-minute captures
public static final String ROOT_GROUP_ID_ALIAS = "root";
public static final String DEFAULT_ROOT_GROUP_NAME = "NiFi Flow";
@@ -245,7 +247,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
private final UserService userService;
private final EventDrivenWorkerQueue eventDrivenWorkerQueue;
private final ComponentStatusRepository componentStatusRepository;
- private final long systemStartTime = System.currentTimeMillis(); // time at which the node was started
+ private final long systemStartTime = System.currentTimeMillis(); // time at which the node was started
private final ConcurrentMap<String, ReportingTaskNode> reportingTasks = new ConcurrentHashMap<>();
// The Heartbeat Bean is used to provide an Atomic Reference to data that is used in heartbeats that may
@@ -336,38 +338,36 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
private final Lock readLock = rwLock.readLock();
private final Lock writeLock = rwLock.writeLock();
- private FlowFileSwapManager flowFileSwapManager; // guarded by read/write lock
-
private static final Logger LOG = LoggerFactory.getLogger(FlowController.class);
private static final Logger heartbeatLogger = LoggerFactory.getLogger("org.apache.nifi.cluster.heartbeat");
public static FlowController createStandaloneInstance(
- final FlowFileEventRepository flowFileEventRepo,
- final NiFiProperties properties,
- final UserService userService,
- final StringEncryptor encryptor) {
+ final FlowFileEventRepository flowFileEventRepo,
+ final NiFiProperties properties,
+ final UserService userService,
+ final StringEncryptor encryptor) {
return new FlowController(
- flowFileEventRepo,
- properties,
- userService,
- encryptor,
- /* configuredForClustering */ false,
- /* NodeProtocolSender */ null);
+ flowFileEventRepo,
+ properties,
+ userService,
+ encryptor,
+ /* configuredForClustering */ false,
+ /* NodeProtocolSender */ null);
}
public static FlowController createClusteredInstance(
- final FlowFileEventRepository flowFileEventRepo,
- final NiFiProperties properties,
- final UserService userService,
- final StringEncryptor encryptor,
- final NodeProtocolSender protocolSender) {
+ final FlowFileEventRepository flowFileEventRepo,
+ final NiFiProperties properties,
+ final UserService userService,
+ final StringEncryptor encryptor,
+ final NodeProtocolSender protocolSender) {
final FlowController flowController = new FlowController(
- flowFileEventRepo,
- properties,
- userService,
- encryptor,
- /* configuredForClustering */ true,
- /* NodeProtocolSender */ protocolSender);
+ flowFileEventRepo,
+ properties,
+ userService,
+ encryptor,
+ /* configuredForClustering */ true,
+ /* NodeProtocolSender */ protocolSender);
flowController.setClusterManagerRemoteSiteInfo(properties.getRemoteInputPort(), properties.isSiteToSiteSecure());
@@ -375,12 +375,12 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
}
private FlowController(
- final FlowFileEventRepository flowFileEventRepo,
- final NiFiProperties properties,
- final UserService userService,
- final StringEncryptor encryptor,
- final boolean configuredForClustering,
- final NodeProtocolSender protocolSender) {
+ final FlowFileEventRepository flowFileEventRepo,
+ final NiFiProperties properties,
+ final UserService userService,
+ final StringEncryptor encryptor,
+ final boolean configuredForClustering,
+ final NodeProtocolSender protocolSender) {
maxTimerDrivenThreads = new AtomicInteger(10);
maxEventDrivenThreads = new AtomicInteger(5);
@@ -416,7 +416,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
final ProcessContextFactory contextFactory = new ProcessContextFactory(contentRepository, flowFileRepository, flowFileEventRepository, counterRepositoryRef.get(), provenanceEventRepository);
processScheduler.setSchedulingAgent(SchedulingStrategy.EVENT_DRIVEN, new EventDrivenSchedulingAgent(
- eventDrivenEngineRef.get(), this, eventDrivenWorkerQueue, contextFactory, maxEventDrivenThreads.get(), encryptor));
+ eventDrivenEngineRef.get(), this, eventDrivenWorkerQueue, contextFactory, maxEventDrivenThreads.get(), encryptor));
final QuartzSchedulingAgent quartzSchedulingAgent = new QuartzSchedulingAgent(this, timerDrivenEngineRef.get(), contextFactory, encryptor);
final TimerDrivenSchedulingAgent timerDrivenAgent = new TimerDrivenSchedulingAgent(this, timerDrivenEngineRef.get(), contextFactory, encryptor);
@@ -468,7 +468,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
externalSiteListener = null;
} else if (isSiteToSiteSecure && sslContext == null) {
LOG.error("Unable to create Secure Site-to-Site Listener because not all required Keystore/Truststore "
- + "Properties are set. Site-to-Site functionality will be disabled until this problem is has been fixed.");
+ + "Properties are set. Site-to-Site functionality will be disabled until this problem is has been fixed.");
externalSiteListener = null;
} else {
// Register the SocketFlowFileServerProtocol as the appropriate resource for site-to-site Server Protocol
@@ -501,7 +501,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
final String implementationClassName = properties.getProperty(NiFiProperties.FLOWFILE_REPOSITORY_IMPLEMENTATION, DEFAULT_FLOWFILE_REPO_IMPLEMENTATION);
if (implementationClassName == null) {
throw new RuntimeException("Cannot create FlowFile Repository because the NiFi Properties is missing the following property: "
- + NiFiProperties.FLOWFILE_REPOSITORY_IMPLEMENTATION);
+ + NiFiProperties.FLOWFILE_REPOSITORY_IMPLEMENTATION);
}
try {
@@ -543,14 +543,22 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
public void initializeFlow() throws IOException {
writeLock.lock();
try {
- flowFileSwapManager = createSwapManager(properties);
+ // get all connections/queues and recover from swap files.
+ final List<Connection> connections = getGroup(getRootGroupId()).findAllConnections();
long maxIdFromSwapFiles = -1L;
- if (flowFileSwapManager != null) {
- if (flowFileRepository.isVolatile()) {
- flowFileSwapManager.purge();
- } else {
- maxIdFromSwapFiles = flowFileSwapManager.recoverSwappedFlowFiles(this, contentClaimManager);
+ if (flowFileRepository.isVolatile()) {
+ for (final Connection connection : connections) {
+ final FlowFileQueue queue = connection.getFlowFileQueue();
+ queue.purgeSwapFiles();
+ }
+ } else {
+ for (final Connection connection : connections) {
+ final FlowFileQueue queue = connection.getFlowFileQueue();
+ final Long maxFlowFileId = queue.recoverSwappedFlowFiles();
+ if (maxFlowFileId != null && maxFlowFileId > maxIdFromSwapFiles) {
+ maxIdFromSwapFiles = maxFlowFileId;
+ }
}
}
@@ -560,10 +568,6 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
// ContentRepository to purge superfluous files
contentRepository.cleanup();
- if (flowFileSwapManager != null) {
- flowFileSwapManager.start(flowFileRepository, this, contentClaimManager, createEventReporter(bulletinRepository));
- }
-
if (externalSiteListener != null) {
externalSiteListener.start();
}
@@ -612,7 +616,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
startConnectable(connectable);
}
} catch (final Throwable t) {
- LOG.error("Unable to start {} due to {}", new Object[]{connectable, t.toString()});
+ LOG.error("Unable to start {} due to {}", new Object[] {connectable, t.toString()});
if (LOG.isDebugEnabled()) {
LOG.error("", t);
}
@@ -627,7 +631,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
remoteGroupPort.getRemoteProcessGroup().startTransmitting(remoteGroupPort);
startedTransmitting++;
} catch (final Throwable t) {
- LOG.error("Unable to start transmitting with {} due to {}", new Object[]{remoteGroupPort, t});
+ LOG.error("Unable to start transmitting with {} due to {}", new Object[] {remoteGroupPort, t});
}
}
@@ -642,7 +646,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
startConnectable(connectable);
}
} catch (final Throwable t) {
- LOG.error("Unable to start {} due to {}", new Object[]{connectable, t});
+ LOG.error("Unable to start {} due to {}", new Object[] {connectable, t});
}
}
@@ -658,7 +662,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
final String implementationClassName = properties.getProperty(NiFiProperties.CONTENT_REPOSITORY_IMPLEMENTATION, DEFAULT_CONTENT_REPO_IMPLEMENTATION);
if (implementationClassName == null) {
throw new RuntimeException("Cannot create Provenance Repository because the NiFi Properties is missing the following property: "
- + NiFiProperties.CONTENT_REPOSITORY_IMPLEMENTATION);
+ + NiFiProperties.CONTENT_REPOSITORY_IMPLEMENTATION);
}
try {
@@ -676,7 +680,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
final String implementationClassName = properties.getProperty(NiFiProperties.PROVENANCE_REPO_IMPLEMENTATION_CLASS, DEFAULT_PROVENANCE_REPO_IMPLEMENTATION);
if (implementationClassName == null) {
throw new RuntimeException("Cannot create Provenance Repository because the NiFi Properties is missing the following property: "
- + NiFiProperties.PROVENANCE_REPO_IMPLEMENTATION_CLASS);
+ + NiFiProperties.PROVENANCE_REPO_IMPLEMENTATION_CLASS);
}
try {
@@ -690,7 +694,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
final String implementationClassName = properties.getProperty(NiFiProperties.COMPONENT_STATUS_REPOSITORY_IMPLEMENTATION, DEFAULT_COMPONENT_STATUS_REPO_IMPLEMENTATION);
if (implementationClassName == null) {
throw new RuntimeException("Cannot create Component Status Repository because the NiFi Properties is missing the following property: "
- + NiFiProperties.COMPONENT_STATUS_REPOSITORY_IMPLEMENTATION);
+ + NiFiProperties.COMPONENT_STATUS_REPOSITORY_IMPLEMENTATION);
}
try {
@@ -721,7 +725,38 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
relationships.add(new Relationship.Builder().name(relationshipName).build());
}
- return builder.id(requireNonNull(id).intern()).name(name == null ? null : name.intern()).relationships(relationships).source(requireNonNull(source)).destination(destination).build();
+ // Create and initialize a FlowFileSwapManager for this connection
+ final FlowFileSwapManager swapManager = createSwapManager(properties);
+ final EventReporter eventReporter = createEventReporter(getBulletinRepository());
+ try (final NarCloseable narCloseable = NarCloseable.withNarLoader()) {
+ final SwapManagerInitializationContext initializationContext = new SwapManagerInitializationContext() {
+ @Override
+ public ResourceClaimManager getResourceClaimManager() {
+ return getResourceClaimManager();
+ }
+
+ @Override
+ public FlowFileRepository getFlowFileRepository() {
+ return flowFileRepository;
+ }
+
+ @Override
+ public EventReporter getEventReporter() {
+ return eventReporter;
+ }
+ };
+
+ swapManager.initialize(initializationContext);
+ }
+
+ return builder.id(requireNonNull(id).intern())
+ .name(name == null ? null : name.intern())
+ .relationships(relationships)
+ .source(requireNonNull(source))
+ .destination(destination)
+ .swapManager(swapManager)
+ .eventReporter(eventReporter)
+ .build();
}
/**
@@ -910,7 +945,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
name = requireNonNull(name).intern();
verifyPortIdDoesNotExist(id);
return new StandardRootGroupPort(id, name, null, TransferDirection.RECEIVE, ConnectableType.INPUT_PORT,
- userService, getBulletinRepository(), processScheduler, Boolean.TRUE.equals(isSiteToSiteSecure));
+ userService, getBulletinRepository(), processScheduler, Boolean.TRUE.equals(isSiteToSiteSecure));
}
/**
@@ -927,7 +962,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
name = requireNonNull(name).intern();
verifyPortIdDoesNotExist(id);
return new StandardRootGroupPort(id, name, null, TransferDirection.SEND, ConnectableType.OUTPUT_PORT,
- userService, getBulletinRepository(), processScheduler, Boolean.TRUE.equals(isSiteToSiteSecure));
+ userService, getBulletinRepository(), processScheduler, Boolean.TRUE.equals(isSiteToSiteSecure));
}
/**
@@ -1083,24 +1118,20 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
try {
flowFileRepository.close();
} catch (final Throwable t) {
- LOG.warn("Unable to shut down FlowFileRepository due to {}", new Object[]{t});
+ LOG.warn("Unable to shut down FlowFileRepository due to {}", new Object[] {t});
}
if (this.timerDrivenEngineRef.get().isTerminated() && eventDrivenEngineRef.get().isTerminated()) {
LOG.info("Controller has been terminated successfully.");
} else {
LOG.warn("Controller hasn't terminated properly. There exists an uninterruptable thread that "
- + "will take an indeterminate amount of time to stop. Might need to kill the program manually.");
+ + "will take an indeterminate amount of time to stop. Might need to kill the program manually.");
}
if (externalSiteListener != null) {
externalSiteListener.stop();
}
- if (flowFileSwapManager != null) {
- flowFileSwapManager.shutdown();
- }
-
if (processScheduler != null) {
processScheduler.shutdown();
}
@@ -1153,7 +1184,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
* @throws FlowSynchronizationException if updates to the controller failed. If this exception is thrown, then the controller should be considered unsafe to be used
*/
public void synchronize(final FlowSynchronizer synchronizer, final DataFlow dataFlow)
- throws FlowSerializationException, FlowSynchronizationException, UninheritableFlowException {
+ throws FlowSerializationException, FlowSynchronizationException, UninheritableFlowException {
writeLock.lock();
try {
LOG.debug("Synchronizing controller with proposed flow");
@@ -1199,7 +1230,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
*
* @param maxThreadCount
*
- * This method must be called while holding the write lock!
+ * This method must be called while holding the write lock!
*/
private void setMaxThreadCount(final int maxThreadCount, final FlowEngine engine, final AtomicInteger maxThreads) {
if (maxThreadCount < 1) {
@@ -1267,7 +1298,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
* @throws ProcessorInstantiationException
*
* @throws IllegalStateException if no process group can be found with the ID of DTO or with the ID of the DTO's parentGroupId, if the template ID specified is invalid, or if the DTO's Parent
- * Group ID changes but the parent group has incoming or outgoing connections
+ * Group ID changes but the parent group has incoming or outgoing connections
*
* @throws NullPointerException if the DTO or its ID is null
*/
@@ -1371,7 +1402,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
*
* @throws NullPointerException if either argument is null
* @throws IllegalStateException if the snippet is not valid because a component in the snippet has an ID that is not unique to this flow, or because it shares an Input Port or Output Port at the
- * root level whose name already exists in the given ProcessGroup, or because the Template contains a Processor or a Prioritizer whose class is not valid within this instance of NiFi.
+ * root level whose name already exists in the given ProcessGroup, or because the Template contains a Processor or a Prioritizer whose class is not valid within this instance of NiFi.
* @throws ProcessorInstantiationException if unable to instantiate a processor
*/
public void instantiateSnippet(final ProcessGroup group, final FlowSnippetDTO dto) throws ProcessorInstantiationException {
@@ -2542,7 +2573,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
if (firstTimeAdded) {
final ComponentLog componentLog = new SimpleProcessLogger(id, taskNode.getReportingTask());
final ReportingInitializationContext config = new StandardReportingInitializationContext(id, taskNode.getName(),
- SchedulingStrategy.TIMER_DRIVEN, "1 min", componentLog, this);
+ SchedulingStrategy.TIMER_DRIVEN, "1 min", componentLog, this);
try {
task.initialize(config);
@@ -2888,7 +2919,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
readLock.lock();
try {
return heartbeatGeneratorFuture != null && !heartbeatGeneratorFuture.isCancelled()
- && heartbeatSenderFuture != null && !heartbeatSenderFuture.isCancelled();
+ && heartbeatSenderFuture != null && !heartbeatSenderFuture.isCancelled();
} finally {
readLock.unlock();
}
@@ -2948,7 +2979,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
/**
* @return the DN of the Cluster Manager that we are currently connected to, if available. This will return null if the instance is not clustered or if the instance is clustered but the NCM's DN
- * is not available - for instance, if cluster communications are not secure
+ * is not available - for instance, if cluster communications are not secure
*/
public String getClusterManagerDN() {
readLock.lock();
@@ -3101,10 +3132,10 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
@Override
public boolean isContentSame() {
return areEqual(event.getPreviousContentClaimContainer(), event.getContentClaimContainer())
- && areEqual(event.getPreviousContentClaimSection(), event.getContentClaimSection())
- && areEqual(event.getPreviousContentClaimIdentifier(), event.getContentClaimIdentifier())
- && areEqual(event.getPreviousContentClaimOffset(), event.getContentClaimOffset())
- && areEqual(event.getPreviousFileSize(), event.getFileSize());
+ && areEqual(event.getPreviousContentClaimSection(), event.getContentClaimSection())
+ && areEqual(event.getPreviousContentClaimIdentifier(), event.getContentClaimIdentifier())
+ && areEqual(event.getPreviousContentClaimOffset(), event.getContentClaimOffset())
+ && areEqual(event.getPreviousFileSize(), event.getFileSize());
}
@Override
@@ -3297,7 +3328,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
// Create the ContentClaim
final ResourceClaim resourceClaim = contentClaimManager.newResourceClaim(event.getPreviousContentClaimContainer(),
- event.getPreviousContentClaimSection(), event.getPreviousContentClaimIdentifier(), false);
+ event.getPreviousContentClaimSection(), event.getPreviousContentClaimIdentifier(), false);
// Increment Claimant Count, since we will now be referencing the Content Claim
contentClaimManager.incrementClaimantCount(resourceClaim);
@@ -3367,7 +3398,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
// Update the FlowFile Repository to indicate that we have added the FlowFile to the flow
final StandardRepositoryRecord record = new StandardRepositoryRecord(queue, flowFileRecord);
record.setDestination(queue);
- flowFileRepository.updateRepository(Collections.<RepositoryRecord>singleton(record));
+ flowFileRepository.updateRepository(Collections.<RepositoryRecord> singleton(record));
// Enqueue the data
queue.put(flowFileRecord);
@@ -3434,11 +3465,9 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
protocolSender.sendBulletins(message);
if (LOG.isDebugEnabled()) {
LOG.debug(
- String.format(
- "Sending bulletins to cluster manager at %s",
- dateFormatter.format(new Date())
- )
- );
+ String.format(
+ "Sending bulletins to cluster manager at %s",
+ dateFormatter.format(new Date())));
}
} catch (final UnknownServiceAddressException usae) {
@@ -3496,7 +3525,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
escapedBulletin = BulletinFactory.createBulletin(bulletin.getCategory(), bulletin.getLevel(), escapedBulletinMessage);
} else {
escapedBulletin = BulletinFactory.createBulletin(bulletin.getGroupId(), bulletin.getSourceId(), bulletin.getSourceType(),
- bulletin.getSourceName(), bulletin.getCategory(), bulletin.getLevel(), escapedBulletinMessage);
+ bulletin.getSourceName(), bulletin.getCategory(), bulletin.getLevel(), escapedBulletinMessage);
}
} else {
escapedBulletin = bulletin;
@@ -3554,9 +3583,9 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
final long sendMillis = TimeUnit.NANOSECONDS.toMillis(sendNanos);
heartbeatLogger.info("Heartbeat created at {} and sent at {}; send took {} millis",
- dateFormatter.format(new Date(message.getHeartbeat().getCreatedTimestamp())),
- dateFormatter.format(new Date()),
- sendMillis);
+ dateFormatter.format(new Date(message.getHeartbeat().getCreatedTimestamp())),
+ dateFormatter.format(new Date()),
+ sendMillis);
} catch (final UnknownServiceAddressException usae) {
if (heartbeatLogger.isDebugEnabled()) {
heartbeatLogger.debug(usae.getMessage());
http://git-wip-us.apache.org/repos/asf/nifi/blob/b8c51dc3/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/BatchingSessionFactory.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/BatchingSessionFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/BatchingSessionFactory.java
index d5dba82..f70f602 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/BatchingSessionFactory.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/BatchingSessionFactory.java
@@ -25,11 +25,11 @@ import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
+import org.apache.nifi.controller.queue.QueueSize;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.processor.FlowFileFilter;
import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.ProcessSessionFactory;
-import org.apache.nifi.processor.QueueSize;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.io.InputStreamCallback;
import org.apache.nifi.processor.io.OutputStreamCallback;
http://git-wip-us.apache.org/repos/asf/nifi/blob/b8c51dc3/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/ConnectionSwapInfo.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/ConnectionSwapInfo.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/ConnectionSwapInfo.java
deleted file mode 100644
index 642e8ff..0000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/ConnectionSwapInfo.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.controller.repository;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.nifi.processor.QueueSize;
-
-public class ConnectionSwapInfo {
-
- private final Map<String, Map<String, QueueSize>> connectionMap = new HashMap<>();
-
- public void addSwapSizeInfo(final String connectionId, final String swapFileLocation, final QueueSize queueSize) {
- Map<String, QueueSize> queueSizeMap = connectionMap.get(connectionId);
- if (queueSizeMap == null) {
- queueSizeMap = new HashMap<>();
- connectionMap.put(connectionId, queueSizeMap);
- }
-
- queueSizeMap.put(swapFileLocation, queueSize);
- }
-
- public Collection<String> getSwapFileLocations(final String connectionId) {
- final Map<String, QueueSize> sizeMap = connectionMap.get(connectionId);
- if (sizeMap == null) {
- return Collections.<String>emptyList();
- }
-
- return Collections.unmodifiableCollection(sizeMap.keySet());
- }
-
- public QueueSize getSwappedSize(final String connectionId, final String swapFileLocation) {
- final Map<String, QueueSize> sizeMap = connectionMap.get(connectionId);
- if (sizeMap == null) {
- return null;
- }
-
- return sizeMap.get(swapFileLocation);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b8c51dc3/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
index 3ba7e4e..a32a485 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
@@ -41,8 +41,9 @@ import java.util.regex.Pattern;
import org.apache.nifi.connectable.Connectable;
import org.apache.nifi.connectable.Connection;
-import org.apache.nifi.controller.FlowFileQueue;
import org.apache.nifi.controller.ProcessorNode;
+import org.apache.nifi.controller.queue.FlowFileQueue;
+import org.apache.nifi.controller.queue.QueueSize;
import org.apache.nifi.controller.repository.claim.ContentClaim;
import org.apache.nifi.controller.repository.claim.ResourceClaim;
import org.apache.nifi.controller.repository.io.ByteCountingInputStream;
@@ -57,7 +58,6 @@ import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.flowfile.attributes.CoreAttributes;
import org.apache.nifi.processor.FlowFileFilter;
import org.apache.nifi.processor.ProcessSession;
-import org.apache.nifi.processor.QueueSize;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.exception.FlowFileAccessException;
import org.apache.nifi.processor.exception.FlowFileHandlingException;
http://git-wip-us.apache.org/repos/asf/nifi/blob/b8c51dc3/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardRepositoryRecord.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardRepositoryRecord.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardRepositoryRecord.java
index 5fcb35a..c5be81e 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardRepositoryRecord.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardRepositoryRecord.java
@@ -20,7 +20,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
-import org.apache.nifi.controller.FlowFileQueue;
+import org.apache.nifi.controller.queue.FlowFileQueue;
import org.apache.nifi.controller.repository.claim.ContentClaim;
import org.apache.nifi.processor.Relationship;
http://git-wip-us.apache.org/repos/asf/nifi/blob/b8c51dc3/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/VolatileFlowFileRepository.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/VolatileFlowFileRepository.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/VolatileFlowFileRepository.java
index a85b23b..ae8824a 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/VolatileFlowFileRepository.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/VolatileFlowFileRepository.java
@@ -21,7 +21,7 @@ import java.util.Collection;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
-import org.apache.nifi.controller.FlowFileQueue;
+import org.apache.nifi.controller.queue.FlowFileQueue;
import org.apache.nifi.controller.repository.claim.ContentClaim;
import org.apache.nifi.controller.repository.claim.ResourceClaim;
import org.apache.nifi.controller.repository.claim.ResourceClaimManager;
http://git-wip-us.apache.org/repos/asf/nifi/blob/b8c51dc3/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/WriteAheadFlowFileRepository.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/WriteAheadFlowFileRepository.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/WriteAheadFlowFileRepository.java
index 5ee5fb5..639a4c8 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/WriteAheadFlowFileRepository.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/WriteAheadFlowFileRepository.java
@@ -41,7 +41,7 @@ import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
-import org.apache.nifi.controller.FlowFileQueue;
+import org.apache.nifi.controller.queue.FlowFileQueue;
import org.apache.nifi.controller.repository.claim.ContentClaim;
import org.apache.nifi.controller.repository.claim.ResourceClaim;
import org.apache.nifi.controller.repository.claim.ResourceClaimManager;
http://git-wip-us.apache.org/repos/asf/nifi/blob/b8c51dc3/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestFileSystemSwapManager.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestFileSystemSwapManager.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestFileSystemSwapManager.java
index b573006..6eeddc5 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestFileSystemSwapManager.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestFileSystemSwapManager.java
@@ -28,10 +28,10 @@ import java.util.Collection;
import java.util.List;
import java.util.concurrent.TimeUnit;
+import org.apache.nifi.controller.queue.FlowFileQueue;
import org.apache.nifi.controller.repository.FlowFileRecord;
import org.apache.nifi.controller.repository.claim.ResourceClaim;
import org.apache.nifi.controller.repository.claim.ResourceClaimManager;
-
import org.junit.Test;
import org.mockito.Mockito;
@@ -42,12 +42,12 @@ public class TestFileSystemSwapManager {
System.setProperty("nifi.properties.file.path", "src/test/resources/nifi.properties");
try (final InputStream fis = new FileInputStream(new File("src/test/resources/old-swap-file.swap"));
- final DataInputStream in = new DataInputStream(new BufferedInputStream(fis))) {
+ final DataInputStream in = new DataInputStream(new BufferedInputStream(fis))) {
final FlowFileQueue flowFileQueue = Mockito.mock(FlowFileQueue.class);
Mockito.when(flowFileQueue.getIdentifier()).thenReturn("87bb99fe-412c-49f6-a441-d1b0af4e20b4");
- final List<FlowFileRecord> records = FileSystemSwapManager.deserializeFlowFiles(in, flowFileQueue, new NopResourceClaimManager());
+ final List<FlowFileRecord> records = FileSystemSwapManager.deserializeFlowFiles(in, flowFileQueue, "/src/test/resources/old-swap-file.swap", new NopResourceClaimManager());
assertEquals(10000, records.size());
for (final FlowFileRecord record : records) {
http://git-wip-us.apache.org/repos/asf/nifi/blob/b8c51dc3/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/TestStandardProcessSession.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/TestStandardProcessSession.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/TestStandardProcessSession.java
index 0e11923..12f8e5e 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/TestStandardProcessSession.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/TestStandardProcessSession.java
@@ -48,9 +48,9 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.nifi.connectable.Connectable;
import org.apache.nifi.connectable.ConnectableType;
import org.apache.nifi.connectable.Connection;
-import org.apache.nifi.controller.FlowFileQueue;
import org.apache.nifi.controller.ProcessScheduler;
import org.apache.nifi.controller.StandardFlowFileQueue;
+import org.apache.nifi.controller.queue.FlowFileQueue;
import org.apache.nifi.controller.repository.claim.ContentClaim;
import org.apache.nifi.controller.repository.claim.ResourceClaim;
import org.apache.nifi.controller.repository.claim.ResourceClaimManager;
@@ -133,7 +133,8 @@ public class TestStandardProcessSession {
final Connection connection = Mockito.mock(Connection.class);
final ProcessScheduler processScheduler = Mockito.mock(ProcessScheduler.class);
- flowFileQueue = new StandardFlowFileQueue("1", connection, processScheduler, 10000);
+ final FlowFileSwapManager swapManager = Mockito.mock(FlowFileSwapManager.class);
+ flowFileQueue = new StandardFlowFileQueue("1", connection, processScheduler, swapManager, null, 10000);
when(connection.getFlowFileQueue()).thenReturn(flowFileQueue);
Mockito.doAnswer(new Answer<Object>() {
@@ -445,7 +446,7 @@ public class TestStandardProcessSession {
session.transfer(newFlowFile, new Relationship.Builder().name("A").build());
session.commit();
- assertEquals(1, provenanceRepo.getEvents(0L, 100000).size()); // 1 event for both parents and children
+ assertEquals(1, provenanceRepo.getEvents(0L, 100000).size()); // 1 event for both parents and children
}
@Test
@@ -809,7 +810,7 @@ public class TestStandardProcessSession {
.entryDate(System.currentTimeMillis())
.contentClaim(new StandardContentClaim(new StandardResourceClaim("x", "x", "0", true), 0L))
- .contentClaimOffset(1000L).size(1L).build();
+ .contentClaimOffset(1000L).size(1L).build();
flowFileQueue.put(flowFileRecord2);
// attempt to read the data.
http://git-wip-us.apache.org/repos/asf/nifi/blob/b8c51dc3/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/TestWriteAheadFlowFileRepository.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/TestWriteAheadFlowFileRepository.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/TestWriteAheadFlowFileRepository.java
index 2138928..e836b44 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/TestWriteAheadFlowFileRepository.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/TestWriteAheadFlowFileRepository.java
@@ -18,6 +18,7 @@ package org.apache.nifi.controller.repository;
import org.apache.nifi.controller.repository.WriteAheadFlowFileRepository;
import org.apache.nifi.controller.repository.StandardRepositoryRecord;
+import org.apache.nifi.controller.queue.FlowFileQueue;
import org.apache.nifi.controller.repository.StandardFlowFileRecord;
import static org.junit.Assert.assertEquals;
import static org.mockito.Matchers.any;
@@ -33,7 +34,6 @@ import java.util.Collection;
import java.util.List;
import org.apache.nifi.connectable.Connection;
-import org.apache.nifi.controller.FlowFileQueue;
import org.apache.nifi.controller.repository.claim.StandardResourceClaimManager;
import org.apache.nifi.util.file.FileUtils;
[13/30] nifi git commit: NIFI-730: bug fixes and code cleanup for
swap manager and flowfile queue
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/77f7d752/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
new file mode 100644
index 0000000..acf2830
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
@@ -0,0 +1,1246 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.PriorityQueue;
+import java.util.Queue;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.nifi.connectable.Connection;
+import org.apache.nifi.controller.queue.DropFlowFileState;
+import org.apache.nifi.controller.queue.DropFlowFileStatus;
+import org.apache.nifi.controller.queue.FlowFileQueue;
+import org.apache.nifi.controller.queue.QueueSize;
+import org.apache.nifi.controller.repository.FlowFileRecord;
+import org.apache.nifi.controller.repository.FlowFileRepository;
+import org.apache.nifi.controller.repository.FlowFileSwapManager;
+import org.apache.nifi.controller.repository.RepositoryRecord;
+import org.apache.nifi.controller.repository.RepositoryRecordType;
+import org.apache.nifi.controller.repository.claim.ContentClaim;
+import org.apache.nifi.controller.repository.claim.ResourceClaim;
+import org.apache.nifi.controller.repository.claim.ResourceClaimManager;
+import org.apache.nifi.events.EventReporter;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.FlowFilePrioritizer;
+import org.apache.nifi.processor.DataUnit;
+import org.apache.nifi.processor.FlowFileFilter;
+import org.apache.nifi.processor.FlowFileFilter.FlowFileFilterResult;
+import org.apache.nifi.provenance.ProvenanceEventBuilder;
+import org.apache.nifi.provenance.ProvenanceEventRecord;
+import org.apache.nifi.provenance.ProvenanceEventRepository;
+import org.apache.nifi.provenance.ProvenanceEventType;
+import org.apache.nifi.reporting.Severity;
+import org.apache.nifi.scheduling.SchedulingStrategy;
+import org.apache.nifi.util.FormatUtils;
+import org.apache.nifi.util.concurrency.TimedLock;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A FlowFileQueue is used to queue FlowFile objects that are awaiting further
+ * processing. Must be thread safe.
+ *
+ */
+public final class StandardFlowFileQueue implements FlowFileQueue {
+
+ public static final int MAX_EXPIRED_RECORDS_PER_ITERATION = 100000;
+ public static final int SWAP_RECORD_POLL_SIZE = 10000;
+
+ private static final Logger logger = LoggerFactory.getLogger(StandardFlowFileQueue.class);
+
+ private PriorityQueue<FlowFileRecord> activeQueue = null;
+ private ArrayList<FlowFileRecord> swapQueue = null;
+
+ // private final AtomicInteger activeQueueSizeRef = new AtomicInteger(0);
+ // private long activeQueueContentSize = 0L;
+ // private int swappedRecordCount = 0;
+ // private long swappedContentSize = 0L;
+ // private final AtomicReference<QueueSize> unacknowledgedSizeRef = new AtomicReference<>(new QueueSize(0, 0L));
+
+ private final AtomicReference<FlowFileQueueSize> size = new AtomicReference<>(new FlowFileQueueSize(0, 0L, 0, 0L, 0, 0L));
+
+ private String maximumQueueDataSize;
+ private long maximumQueueByteCount;
+ private boolean swapMode = false;
+ private long maximumQueueObjectCount;
+
+ private final EventReporter eventReporter;
+ private final AtomicLong flowFileExpirationMillis;
+ private final Connection connection;
+ private final AtomicReference<String> flowFileExpirationPeriod;
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true);
+ private final List<FlowFilePrioritizer> priorities;
+ private final int swapThreshold;
+ private final FlowFileSwapManager swapManager;
+ private final List<String> swapLocations = new ArrayList<>();
+ private final TimedLock readLock;
+ private final TimedLock writeLock;
+ private final String identifier;
+ private final FlowFileRepository flowFileRepository;
+ private final ProvenanceEventRepository provRepository;
+ private final ResourceClaimManager resourceClaimManager;
+
+ private final AtomicBoolean queueFullRef = new AtomicBoolean(false);
+
+ // SCHEDULER CANNOT BE NOTIFIED OF EVENTS WITH THE WRITE LOCK HELD! DOING SO WILL RESULT IN A DEADLOCK!
+ private final ProcessScheduler scheduler;
+
+ public StandardFlowFileQueue(final String identifier, final Connection connection, final FlowFileRepository flowFileRepo, final ProvenanceEventRepository provRepo,
+ final ResourceClaimManager resourceClaimManager, final ProcessScheduler scheduler, final FlowFileSwapManager swapManager, final EventReporter eventReporter, final int swapThreshold) {
+ activeQueue = new PriorityQueue<>(20, new Prioritizer(new ArrayList<FlowFilePrioritizer>()));
+ priorities = new ArrayList<>();
+ maximumQueueObjectCount = 0L;
+ maximumQueueDataSize = "0 MB";
+ maximumQueueByteCount = 0L;
+ flowFileExpirationMillis = new AtomicLong(0);
+ flowFileExpirationPeriod = new AtomicReference<>("0 mins");
+ swapQueue = new ArrayList<>();
+ this.eventReporter = eventReporter;
+ this.swapManager = swapManager;
+ this.flowFileRepository = flowFileRepo;
+ this.provRepository = provRepo;
+ this.resourceClaimManager = resourceClaimManager;
+
+ this.identifier = identifier;
+ this.swapThreshold = swapThreshold;
+ this.scheduler = scheduler;
+ this.connection = connection;
+
+ readLock = new TimedLock(this.lock.readLock(), identifier + " Read Lock", 100);
+ writeLock = new TimedLock(this.lock.writeLock(), identifier + " Write Lock", 100);
+ }
+
+ @Override
+ public String getIdentifier() {
+ return identifier;
+ }
+
+ @Override
+ public List<FlowFilePrioritizer> getPriorities() {
+ return Collections.unmodifiableList(priorities);
+ }
+
+ @Override
+ public void setPriorities(final List<FlowFilePrioritizer> newPriorities) {
+ writeLock.lock();
+ try {
+ final PriorityQueue<FlowFileRecord> newQueue = new PriorityQueue<>(Math.max(20, activeQueue.size()), new Prioritizer(newPriorities));
+ newQueue.addAll(activeQueue);
+ activeQueue = newQueue;
+ priorities.clear();
+ priorities.addAll(newPriorities);
+ } finally {
+ writeLock.unlock("setPriorities");
+ }
+ }
+
+ @Override
+ public void setBackPressureObjectThreshold(final long maxQueueSize) {
+ writeLock.lock();
+ try {
+ maximumQueueObjectCount = maxQueueSize;
+ this.queueFullRef.set(determineIfFull());
+ } finally {
+ writeLock.unlock("setBackPressureObjectThreshold");
+ }
+ }
+
+ @Override
+ public long getBackPressureObjectThreshold() {
+ readLock.lock();
+ try {
+ return maximumQueueObjectCount;
+ } finally {
+ readLock.unlock("getBackPressureObjectThreshold");
+ }
+ }
+
+ @Override
+ public void setBackPressureDataSizeThreshold(final String maxDataSize) {
+ writeLock.lock();
+ try {
+ maximumQueueByteCount = DataUnit.parseDataSize(maxDataSize, DataUnit.B).longValue();
+ maximumQueueDataSize = maxDataSize;
+ this.queueFullRef.set(determineIfFull());
+ } finally {
+ writeLock.unlock("setBackPressureDataSizeThreshold");
+ }
+ }
+
+ @Override
+ public String getBackPressureDataSizeThreshold() {
+ readLock.lock();
+ try {
+ return maximumQueueDataSize;
+ } finally {
+ readLock.unlock("getBackPressureDataSizeThreshold");
+ }
+ }
+
+ @Override
+ public QueueSize size() {
+ return getQueueSize();
+ }
+
+
+ private QueueSize getQueueSize() {
+ return size.get().toQueueSize();
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return size.get().isEmpty();
+ }
+
+ @Override
+ public boolean isActiveQueueEmpty() {
+ return size.get().activeQueueCount == 0;
+ }
+
+ public QueueSize getActiveQueueSize() {
+ return size.get().activeQueueSize();
+ }
+
+ @Override
+ public void acknowledge(final FlowFileRecord flowFile) {
+ if (queueFullRef.get()) {
+ writeLock.lock();
+ try {
+ incrementUnacknowledgedQueueSize(-1, -flowFile.getSize());
+ queueFullRef.set(determineIfFull());
+ } finally {
+ writeLock.unlock("acknowledge(FlowFileRecord)");
+ }
+ } else {
+ incrementUnacknowledgedQueueSize(-1, -flowFile.getSize());
+ }
+
+ if (connection.getSource().getSchedulingStrategy() == SchedulingStrategy.EVENT_DRIVEN) {
+ // queue was full but no longer is. Notify that the source may now be available to run,
+ // because of back pressure caused by this queue.
+ scheduler.registerEvent(connection.getSource());
+ }
+ }
+
+ @Override
+ public void acknowledge(final Collection<FlowFileRecord> flowFiles) {
+ long totalSize = 0L;
+ for (final FlowFileRecord flowFile : flowFiles) {
+ totalSize += flowFile.getSize();
+ }
+
+ if (queueFullRef.get()) {
+ writeLock.lock();
+ try {
+ incrementUnacknowledgedQueueSize(-flowFiles.size(), -totalSize);
+ queueFullRef.set(determineIfFull());
+ } finally {
+ writeLock.unlock("acknowledge(FlowFileRecord)");
+ }
+ } else {
+ incrementUnacknowledgedQueueSize(-flowFiles.size(), -totalSize);
+ }
+
+ if (connection.getSource().getSchedulingStrategy() == SchedulingStrategy.EVENT_DRIVEN) {
+ // it's possible that queue was full but no longer is. Notify that the source may now be available to run,
+ // because of back pressure caused by this queue.
+ scheduler.registerEvent(connection.getSource());
+ }
+ }
+
+ @Override
+ public boolean isFull() {
+ return queueFullRef.get();
+ }
+
+ /**
+ * MUST be called with either the read or write lock held
+ *
+ * @return true if full
+ */
+ private boolean determineIfFull() {
+ final long maxSize = maximumQueueObjectCount;
+ final long maxBytes = maximumQueueByteCount;
+ if (maxSize <= 0 && maxBytes <= 0) {
+ return false;
+ }
+
+ final QueueSize queueSize = getQueueSize();
+ if (maxSize > 0 && queueSize.getObjectCount() >= maxSize) {
+ return true;
+ }
+
+ if (maxBytes > 0 && queueSize.getByteCount() >= maxBytes) {
+ return true;
+ }
+
+ return false;
+ }
+
+ @Override
+ public void put(final FlowFileRecord file) {
+ writeLock.lock();
+ try {
+ if (swapMode || activeQueue.size() >= swapThreshold) {
+ swapQueue.add(file);
+ incrementSwapQueueSize(1, file.getSize());
+ swapMode = true;
+ writeSwapFilesIfNecessary();
+ } else {
+ incrementActiveQueueSize(1, file.getSize());
+ activeQueue.add(file);
+ }
+
+ queueFullRef.set(determineIfFull());
+ } finally {
+ writeLock.unlock("put(FlowFileRecord)");
+ }
+
+ if (connection.getDestination().getSchedulingStrategy() == SchedulingStrategy.EVENT_DRIVEN) {
+ scheduler.registerEvent(connection.getDestination());
+ }
+ }
+
+ @Override
+ public void putAll(final Collection<FlowFileRecord> files) {
+ final int numFiles = files.size();
+ long bytes = 0L;
+ for (final FlowFile flowFile : files) {
+ bytes += flowFile.getSize();
+ }
+
+ writeLock.lock();
+ try {
+ if (swapMode || activeQueue.size() >= swapThreshold - numFiles) {
+ swapQueue.addAll(files);
+ incrementSwapQueueSize(numFiles, bytes);
+ swapMode = true;
+ writeSwapFilesIfNecessary();
+ } else {
+ incrementActiveQueueSize(numFiles, bytes);
+ activeQueue.addAll(files);
+ }
+
+ queueFullRef.set(determineIfFull());
+ } finally {
+ writeLock.unlock("putAll");
+ }
+
+ if (connection.getDestination().getSchedulingStrategy() == SchedulingStrategy.EVENT_DRIVEN) {
+ scheduler.registerEvent(connection.getDestination());
+ }
+ }
+
+
+ private boolean isLaterThan(final Long maxAge) {
+ if (maxAge == null) {
+ return false;
+ }
+ return maxAge < System.currentTimeMillis();
+ }
+
+ private Long getExpirationDate(final FlowFile flowFile, final long expirationMillis) {
+ if (flowFile == null) {
+ return null;
+ }
+ if (expirationMillis <= 0) {
+ return null;
+ } else {
+ final long entryDate = flowFile.getEntryDate();
+ final long expirationDate = entryDate + expirationMillis;
+ return expirationDate;
+ }
+ }
+
+ @Override
+ public FlowFileRecord poll(final Set<FlowFileRecord> expiredRecords) {
+ FlowFileRecord flowFile = null;
+
+ // First check if we have any records Pre-Fetched.
+ final long expirationMillis = flowFileExpirationMillis.get();
+ writeLock.lock();
+ try {
+ flowFile = doPoll(expiredRecords, expirationMillis);
+ return flowFile;
+ } finally {
+ writeLock.unlock("poll(Set)");
+
+ if (flowFile != null) {
+ incrementUnacknowledgedQueueSize(1, flowFile.getSize());
+ }
+ }
+ }
+
+ private FlowFileRecord doPoll(final Set<FlowFileRecord> expiredRecords, final long expirationMillis) {
+ FlowFileRecord flowFile;
+ boolean isExpired;
+
+ migrateSwapToActive();
+ final boolean queueFullAtStart = queueFullRef.get();
+
+ int expiredRecordCount = 0;
+ long expiredBytes = 0L;
+
+ do {
+ flowFile = this.activeQueue.poll();
+
+ isExpired = isLaterThan(getExpirationDate(flowFile, expirationMillis));
+ if (isExpired) {
+ expiredRecords.add(flowFile);
+ expiredRecordCount++;
+ expiredBytes += flowFile.getSize();
+
+ if (expiredRecords.size() >= MAX_EXPIRED_RECORDS_PER_ITERATION) {
+ break;
+ }
+ } else if (flowFile != null && flowFile.isPenalized()) {
+ this.activeQueue.add(flowFile);
+ flowFile = null;
+ break;
+ }
+
+ if (flowFile != null) {
+ incrementActiveQueueSize(-1, -flowFile.getSize());
+ }
+
+ if (expiredRecordCount > 0) {
+ incrementActiveQueueSize(-expiredRecordCount, -expiredBytes);
+ }
+ } while (isExpired);
+
+ // if at least 1 FlowFile was expired & the queue was full before we started, then
+ // we need to determine whether or not the queue is full again. If no FlowFile was expired,
+ // then the queue will still be full until the appropriate #acknowledge method is called.
+ if (queueFullAtStart && !expiredRecords.isEmpty()) {
+ queueFullRef.set(determineIfFull());
+ }
+
+ return isExpired ? null : flowFile;
+ }
+
+ @Override
+ public List<FlowFileRecord> poll(int maxResults, final Set<FlowFileRecord> expiredRecords) {
+ final List<FlowFileRecord> records = new ArrayList<>(Math.min(1024, maxResults));
+
+ // First check if we have any records Pre-Fetched.
+ writeLock.lock();
+ try {
+ doPoll(records, maxResults, expiredRecords);
+ } finally {
+ writeLock.unlock("poll(int, Set)");
+ }
+ return records;
+ }
+
+ private void doPoll(final List<FlowFileRecord> records, int maxResults, final Set<FlowFileRecord> expiredRecords) {
+ migrateSwapToActive();
+
+ final boolean queueFullAtStart = queueFullRef.get();
+
+ final long bytesDrained = drainQueue(activeQueue, records, maxResults, expiredRecords);
+
+ long expiredBytes = 0L;
+ for (final FlowFileRecord record : expiredRecords) {
+ expiredBytes += record.getSize();
+ }
+
+ incrementActiveQueueSize(-(expiredRecords.size() + records.size()), -bytesDrained);
+ incrementUnacknowledgedQueueSize(records.size(), bytesDrained - expiredBytes);
+
+ // if at least 1 FlowFile was expired & the queue was full before we started, then
+ // we need to determine whether or not the queue is full again. If no FlowFile was expired,
+ // then the queue will still be full until the appropriate #acknowledge method is called.
+ if (queueFullAtStart && !expiredRecords.isEmpty()) {
+ queueFullRef.set(determineIfFull());
+ }
+ }
+
+ /**
+ * If there are FlowFiles waiting on the swap queue, move them to the active
+ * queue until we meet our threshold. This prevents us from having to swap
+ * them to disk & then back out.
+ *
+ * This method MUST be called with the writeLock held.
+ */
+ private void migrateSwapToActive() {
+ // Migrate as many FlowFiles as we can from the Swap Queue to the Active Queue, so that we don't
+ // have to swap them out & then swap them back in.
+ // If we don't do this, we could get into a situation where we have potentially thousands of FlowFiles
+ // sitting on the Swap Queue but not getting processed because there aren't enough to be swapped out.
+ // In particular, this can happen if the queue is typically filled with surges.
+ // For example, if the queue has 25,000 FlowFiles come in, it may process 20,000 of them and leave
+ // 5,000 sitting on the Swap Queue. If it then takes an hour for an additional 5,000 FlowFiles to come in,
+ // those FlowFiles sitting on the Swap Queue will sit there for an hour, waiting to be swapped out and
+ // swapped back in again.
+ // Calling this method when records are polled prevents this condition by migrating FlowFiles from the
+ // Swap Queue to the Active Queue. However, we don't do this if there are FlowFiles already swapped out
+ // to disk, because we want them to be swapped back in in the same order that they were swapped out.
+
+ if (activeQueue.size() > swapThreshold - SWAP_RECORD_POLL_SIZE) {
+ return;
+ }
+
+ // If there are swap files waiting to be swapped in, swap those in first. We do this in order to ensure that those that
+ // were swapped out first are then swapped back in first. If we instead just immediately migrated the FlowFiles from the
+ // swap queue to the active queue, and we never run out of FlowFiles in the active queue (because destination cannot
+ // keep up with queue), we will end up always processing the new FlowFiles first instead of the FlowFiles that arrived
+ // first.
+ if (!swapLocations.isEmpty()) {
+ final String swapLocation = swapLocations.remove(0);
+ try {
+ final List<FlowFileRecord> swappedIn = swapManager.swapIn(swapLocation, this);
+ long swapSize = 0L;
+ for (final FlowFileRecord flowFile : swappedIn) {
+ swapSize += flowFile.getSize();
+ }
+ incrementSwapQueueSize(-swappedIn.size(), -swapSize);
+ incrementActiveQueueSize(swappedIn.size(), swapSize);
+ activeQueue.addAll(swappedIn);
+ return;
+ } catch (final FileNotFoundException fnfe) {
+ logger.error("Failed to swap in FlowFiles from Swap File {} because the Swap File can no longer be found", swapLocation);
+ if (eventReporter != null) {
+ eventReporter.reportEvent(Severity.ERROR, "Swap File", "Failed to swap in FlowFiles from Swap File " + swapLocation + " because the Swap File can no longer be found");
+ }
+ return;
+ } catch (final IOException ioe) {
+ logger.error("Failed to swap in FlowFiles from Swap File {}; Swap File appears to be corrupt!", swapLocation);
+ logger.error("", ioe);
+ if (eventReporter != null) {
+ eventReporter.reportEvent(Severity.ERROR, "Swap File", "Failed to swap in FlowFiles from Swap File " +
+ swapLocation + "; Swap File appears to be corrupt! Some FlowFiles in the queue may not be accessible. See logs for more information.");
+ }
+ return;
+ }
+ }
+
+ // this is the most common condition (nothing is swapped out), so do the check first and avoid the expense
+ // of other checks for 99.999% of the cases.
+ if (size.get().swappedCount == 0 && swapQueue.isEmpty()) {
+ return;
+ }
+
+ if (size.get().swappedCount > swapQueue.size()) {
+ // we already have FlowFiles swapped out, so we won't migrate the queue; we will wait for
+ // an external process to swap FlowFiles back in.
+ return;
+ }
+
+ int recordsMigrated = 0;
+ long bytesMigrated = 0L;
+ final Iterator<FlowFileRecord> swapItr = swapQueue.iterator();
+ while (activeQueue.size() < swapThreshold && swapItr.hasNext()) {
+ final FlowFileRecord toMigrate = swapItr.next();
+ activeQueue.add(toMigrate);
+ bytesMigrated += toMigrate.getSize();
+ recordsMigrated++;
+ swapItr.remove();
+ }
+
+ if (recordsMigrated > 0) {
+ incrementActiveQueueSize(recordsMigrated, bytesMigrated);
+ incrementSwapQueueSize(-recordsMigrated, -bytesMigrated);
+ }
+
+ if (size.get().swappedCount == 0) {
+ swapMode = false;
+ }
+ }
+
+ /**
+ * This method MUST be called with the write lock held
+ */
+ private void writeSwapFilesIfNecessary() {
+ if (swapQueue.size() < SWAP_RECORD_POLL_SIZE) {
+ return;
+ }
+
+ final int numSwapFiles = swapQueue.size() / SWAP_RECORD_POLL_SIZE;
+
+ int originalSwapQueueCount = swapQueue.size();
+ long originalSwapQueueBytes = 0L;
+ for (final FlowFileRecord flowFile : swapQueue) {
+ originalSwapQueueBytes += flowFile.getSize();
+ }
+
+ // Create a new Priority queue with the prioritizers that are set, but reverse the
+ // prioritizers because we want to pull the lowest-priority FlowFiles to swap out
+ final PriorityQueue<FlowFileRecord> tempQueue = new PriorityQueue<>(activeQueue.size() + swapQueue.size(), Collections.reverseOrder(new Prioritizer(priorities)));
+ tempQueue.addAll(activeQueue);
+ tempQueue.addAll(swapQueue);
+
+ long bytesSwappedOut = 0L;
+ int flowFilesSwappedOut = 0;
+ final List<String> swapLocations = new ArrayList<>(numSwapFiles);
+ for (int i = 0; i < numSwapFiles; i++) {
+ // Create a new swap file for the next SWAP_RECORD_POLL_SIZE records
+ final List<FlowFileRecord> toSwap = new ArrayList<>(SWAP_RECORD_POLL_SIZE);
+ for (int j = 0; j < SWAP_RECORD_POLL_SIZE; j++) {
+ final FlowFileRecord flowFile = tempQueue.poll();
+ toSwap.add(flowFile);
+ bytesSwappedOut += flowFile.getSize();
+ flowFilesSwappedOut++;
+ }
+
+ try {
+ Collections.reverse(toSwap); // currently ordered in reverse priority order based on the ordering of the temp queue.
+ final String swapLocation = swapManager.swapOut(toSwap, this);
+ swapLocations.add(swapLocation);
+ } catch (final IOException ioe) {
+ tempQueue.addAll(toSwap); // if we failed, we must add the FlowFiles back to the queue.
+ logger.error("FlowFile Queue with identifier {} has {} FlowFiles queued up. Attempted to spill FlowFile information over to disk in order to avoid exhausting "
+ + "the Java heap space but failed to write information to disk due to {}", getIdentifier(), getQueueSize().getObjectCount(), ioe.toString());
+ logger.error("", ioe);
+ if (eventReporter != null) {
+ eventReporter.reportEvent(Severity.ERROR, "Failed to Overflow to Disk", "Flowfile Queue with identifier " + getIdentifier() + " has " + getQueueSize().getObjectCount() +
+ " queued up. Attempted to spill FlowFile information over to disk in order to avoid exhausting the Java heap space but failed to write information to disk. "
+ + "See logs for more information.");
+ }
+
+ break;
+ }
+ }
+
+ // Pull any records off of the temp queue that won't fit back on the active queue, and add those to the
+ // swap queue. Then add the records back to the active queue.
+ swapQueue.clear();
+ long updatedSwapQueueBytes = 0L;
+ while (tempQueue.size() > swapThreshold) {
+ final FlowFileRecord record = tempQueue.poll();
+ swapQueue.add(record);
+ updatedSwapQueueBytes += record.getSize();
+ }
+
+ Collections.reverse(swapQueue); // currently ordered in reverse priority order based on the ordering of the temp queue
+
+ // replace the contents of the active queue, since we've merged it with the swap queue.
+ activeQueue.clear();
+ FlowFileRecord toRequeue;
+ long activeQueueBytes = 0L;
+ while ((toRequeue = tempQueue.poll()) != null) {
+ activeQueue.offer(toRequeue);
+ activeQueueBytes += toRequeue.getSize();
+ }
+
+ boolean updated = false;
+ while (!updated) {
+ final FlowFileQueueSize originalSize = size.get();
+
+ final int addedSwapRecords = swapQueue.size() - originalSwapQueueCount;
+ final long addedSwapBytes = updatedSwapQueueBytes - originalSwapQueueBytes;
+
+ final FlowFileQueueSize newSize = new FlowFileQueueSize(activeQueue.size(), activeQueueBytes,
+ originalSize.swappedCount + addedSwapRecords + flowFilesSwappedOut, originalSize.swappedBytes + addedSwapBytes + bytesSwappedOut,
+ originalSize.unacknowledgedCount, originalSize.unacknowledgedBytes);
+ updated = size.compareAndSet(originalSize, newSize);
+ }
+
+ this.swapLocations.addAll(swapLocations);
+ }
+
+
+ @Override
+ public long drainQueue(final Queue<FlowFileRecord> sourceQueue, final List<FlowFileRecord> destination, int maxResults, final Set<FlowFileRecord> expiredRecords) {
+ long drainedSize = 0L;
+ FlowFileRecord pulled = null;
+
+ final long expirationMillis = this.flowFileExpirationMillis.get();
+ while (destination.size() < maxResults && (pulled = sourceQueue.poll()) != null) {
+ if (isLaterThan(getExpirationDate(pulled, expirationMillis))) {
+ expiredRecords.add(pulled);
+ if (expiredRecords.size() >= MAX_EXPIRED_RECORDS_PER_ITERATION) {
+ break;
+ }
+ } else {
+ if (pulled.isPenalized()) {
+ sourceQueue.add(pulled);
+ break;
+ }
+ destination.add(pulled);
+ }
+ drainedSize += pulled.getSize();
+ }
+ return drainedSize;
+ }
+
+ @Override
+ public List<FlowFileRecord> poll(final FlowFileFilter filter, final Set<FlowFileRecord> expiredRecords) {
+ long bytesPulled = 0L;
+ int flowFilesPulled = 0;
+
+ writeLock.lock();
+ try {
+ migrateSwapToActive();
+
+ final long expirationMillis = this.flowFileExpirationMillis.get();
+ final boolean queueFullAtStart = queueFullRef.get();
+
+ final List<FlowFileRecord> selectedFlowFiles = new ArrayList<>();
+ final List<FlowFileRecord> unselected = new ArrayList<>();
+
+ while (true) {
+ FlowFileRecord flowFile = this.activeQueue.poll();
+ if (flowFile == null) {
+ break;
+ }
+
+ final boolean isExpired = isLaterThan(getExpirationDate(flowFile, expirationMillis));
+ if (isExpired) {
+ expiredRecords.add(flowFile);
+ bytesPulled += flowFile.getSize();
+ flowFilesPulled++;
+
+ if (expiredRecords.size() >= MAX_EXPIRED_RECORDS_PER_ITERATION) {
+ break;
+ } else {
+ continue;
+ }
+ } else if (flowFile.isPenalized()) {
+ this.activeQueue.add(flowFile);
+ flowFile = null;
+ break; // just stop searching because the rest are all penalized.
+ }
+
+ final FlowFileFilterResult result = filter.filter(flowFile);
+ if (result.isAccept()) {
+ bytesPulled += flowFile.getSize();
+ flowFilesPulled++;
+
+ incrementUnacknowledgedQueueSize(1, flowFile.getSize());
+ selectedFlowFiles.add(flowFile);
+ } else {
+ unselected.add(flowFile);
+ }
+
+ if (!result.isContinue()) {
+ break;
+ }
+ }
+
+ this.activeQueue.addAll(unselected);
+
+ // if at least 1 FlowFile was expired & the queue was full before we started, then
+ // we need to determine whether or not the queue is full again. If no FlowFile was expired,
+ // then the queue will still be full until the appropriate #acknowledge method is called.
+ if (queueFullAtStart && !expiredRecords.isEmpty()) {
+ queueFullRef.set(determineIfFull());
+ }
+
+ return selectedFlowFiles;
+ } finally {
+ incrementActiveQueueSize(-flowFilesPulled, -bytesPulled);
+ writeLock.unlock("poll(Filter, Set)");
+ }
+ }
+
+
+
+ private static final class Prioritizer implements Comparator<FlowFileRecord>, Serializable {
+
+ private static final long serialVersionUID = 1L;
+ private final transient List<FlowFilePrioritizer> prioritizers = new ArrayList<>();
+
+ private Prioritizer(final List<FlowFilePrioritizer> priorities) {
+ if (null != priorities) {
+ prioritizers.addAll(priorities);
+ }
+ }
+
+ @Override
+ public int compare(final FlowFileRecord f1, final FlowFileRecord f2) {
+ int returnVal = 0;
+ final boolean f1Penalized = f1.isPenalized();
+ final boolean f2Penalized = f2.isPenalized();
+
+ if (f1Penalized && !f2Penalized) {
+ return 1;
+ } else if (!f1Penalized && f2Penalized) {
+ return -1;
+ }
+
+ if (f1Penalized && f2Penalized) {
+ if (f1.getPenaltyExpirationMillis() < f2.getPenaltyExpirationMillis()) {
+ return -1;
+ } else if (f1.getPenaltyExpirationMillis() > f2.getPenaltyExpirationMillis()) {
+ return 1;
+ }
+ }
+
+ if (!prioritizers.isEmpty()) {
+ for (final FlowFilePrioritizer prioritizer : prioritizers) {
+ returnVal = prioritizer.compare(f1, f2);
+ if (returnVal != 0) {
+ return returnVal;
+ }
+ }
+ }
+
+ final ContentClaim claim1 = f1.getContentClaim();
+ final ContentClaim claim2 = f2.getContentClaim();
+
+ // put the one without a claim first
+ if (claim1 == null && claim2 != null) {
+ return -1;
+ } else if (claim1 != null && claim2 == null) {
+ return 1;
+ } else if (claim1 != null && claim2 != null) {
+ final int claimComparison = claim1.compareTo(claim2);
+ if (claimComparison != 0) {
+ return claimComparison;
+ }
+
+ final int claimOffsetComparison = Long.compare(f1.getContentClaimOffset(), f2.getContentClaimOffset());
+ if (claimOffsetComparison != 0) {
+ return claimOffsetComparison;
+ }
+ }
+
+ return Long.compare(f1.getId(), f2.getId());
+ }
+ }
+
+ @Override
+ public String getFlowFileExpiration() {
+ return flowFileExpirationPeriod.get();
+ }
+
+ @Override
+ public int getFlowFileExpiration(final TimeUnit timeUnit) {
+ return (int) timeUnit.convert(flowFileExpirationMillis.get(), TimeUnit.MILLISECONDS);
+ }
+
+ @Override
+ public void setFlowFileExpiration(final String flowExpirationPeriod) {
+ final long millis = FormatUtils.getTimeDuration(flowExpirationPeriod, TimeUnit.MILLISECONDS);
+ if (millis < 0) {
+ throw new IllegalArgumentException("FlowFile Expiration Period must be positive");
+ }
+ this.flowFileExpirationPeriod.set(flowExpirationPeriod);
+ this.flowFileExpirationMillis.set(millis);
+ }
+
+
+ @Override
+ public void purgeSwapFiles() {
+ swapManager.purge();
+ }
+
+ @Override
+ public Long recoverSwappedFlowFiles() {
+ int swapFlowFileCount = 0;
+ long swapByteCount = 0L;
+ Long maxId = null;
+
+ writeLock.lock();
+ try {
+ final List<String> swapLocations;
+ try {
+ swapLocations = swapManager.recoverSwapLocations(this);
+ } catch (final IOException ioe) {
+ logger.error("Failed to determine whether or not any Swap Files exist for FlowFile Queue {}", getIdentifier());
+ logger.error("", ioe);
+ if (eventReporter != null) {
+ eventReporter.reportEvent(Severity.ERROR, "FlowFile Swapping", "Failed to determine whether or not any Swap Files exist for FlowFile Queue " +
+ getIdentifier() + "; see logs for more detials");
+ }
+ return null;
+ }
+
+ for (final String swapLocation : swapLocations) {
+ try {
+ final QueueSize queueSize = swapManager.getSwapSize(swapLocation);
+ final Long maxSwapRecordId = swapManager.getMaxRecordId(swapLocation);
+ if (maxSwapRecordId != null) {
+ if (maxId == null || maxSwapRecordId > maxId) {
+ maxId = maxSwapRecordId;
+ }
+ }
+
+ swapFlowFileCount += queueSize.getObjectCount();
+ swapByteCount += queueSize.getByteCount();
+ } catch (final IOException ioe) {
+ logger.error("Failed to recover FlowFiles from Swap File {}; the file appears to be corrupt", swapLocation, ioe.toString());
+ logger.error("", ioe);
+ if (eventReporter != null) {
+ eventReporter.reportEvent(Severity.ERROR, "FlowFile Swapping", "Failed to recover FlowFiles from Swap File " + swapLocation +
+ "; the file appears to be corrupt. See logs for more details");
+ }
+ }
+ }
+
+ incrementSwapQueueSize(swapFlowFileCount, swapByteCount);
+ this.swapLocations.addAll(swapLocations);
+ } finally {
+ writeLock.unlock("Recover Swap Files");
+ }
+
+ return maxId;
+ }
+
+
+ @Override
+ public String toString() {
+ return "FlowFileQueue[id=" + identifier + "]";
+ }
+
+ private final ConcurrentMap<String, DropFlowFileRequest> dropRequestMap = new ConcurrentHashMap<>();
+
+ @Override
+ public DropFlowFileStatus dropFlowFiles(final String requestIdentifier, final String requestor) {
+ logger.info("Initiating drop of FlowFiles from {} on behalf of {} (request identifier={})", this, requestor, requestIdentifier);
+
+ // purge any old requests from the map just to keep it clean. But if there are very requests, which is usually the case, then don't bother
+ if (dropRequestMap.size() > 10) {
+ final List<String> toDrop = new ArrayList<>();
+ for (final Map.Entry<String, DropFlowFileRequest> entry : dropRequestMap.entrySet()) {
+ final DropFlowFileRequest request = entry.getValue();
+ final boolean completed = request.getState() == DropFlowFileState.COMPLETE || request.getState() == DropFlowFileState.FAILURE;
+
+ if (completed && System.currentTimeMillis() - request.getLastUpdated() > TimeUnit.MINUTES.toMillis(5L)) {
+ toDrop.add(entry.getKey());
+ }
+ }
+
+ for (final String requestId : toDrop) {
+ dropRequestMap.remove(requestId);
+ }
+ }
+
+ final DropFlowFileRequest dropRequest = new DropFlowFileRequest(requestIdentifier);
+ final Thread t = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ writeLock.lock();
+ try {
+ dropRequest.setState(DropFlowFileState.DROPPING_ACTIVE_FLOWFILES);
+ logger.debug("For DropFlowFileRequest {}, original size is {}", requestIdentifier, getQueueSize());
+ dropRequest.setOriginalSize(getQueueSize());
+
+ try {
+ final List<FlowFileRecord> activeQueueRecords = new ArrayList<>(activeQueue);
+
+ QueueSize droppedSize;
+ try {
+ droppedSize = drop(activeQueueRecords, requestor);
+ logger.debug("For DropFlowFileRequest {}, Dropped {} from active queue", requestIdentifier, droppedSize);
+ } catch (final IOException ioe) {
+ logger.error("Failed to drop the FlowFiles from queue {} due to {}", StandardFlowFileQueue.this.getIdentifier(), ioe.toString());
+ logger.error("", ioe);
+
+ dropRequest.setState(DropFlowFileState.FAILURE, "Failed to drop FlowFiles due to " + ioe.toString());
+ return;
+ }
+
+ activeQueue.clear();
+ incrementActiveQueueSize(-droppedSize.getObjectCount(), -droppedSize.getByteCount());
+ dropRequest.setCurrentSize(getQueueSize());
+ dropRequest.setDroppedSize(dropRequest.getDroppedSize().add(droppedSize));
+
+ try {
+ final QueueSize swapSize = size.get().swapQueueSize();
+
+ logger.debug("For DropFlowFileRequest {}, Swap Queue has {} elements, Swapped Record Count = {}, Swapped Content Size = {}",
+ requestIdentifier, swapQueue.size(), swapSize.getObjectCount(), swapSize.getByteCount());
+ droppedSize = drop(swapQueue, requestor);
+ } catch (final IOException ioe) {
+ logger.error("Failed to drop the FlowFiles from queue {} due to {}", StandardFlowFileQueue.this.getIdentifier(), ioe.toString());
+ logger.error("", ioe);
+
+ dropRequest.setState(DropFlowFileState.FAILURE, "Failed to drop FlowFiles due to " + ioe.toString());
+ return;
+ }
+
+ swapQueue.clear();
+ dropRequest.setCurrentSize(getQueueSize());
+ dropRequest.setDroppedSize(dropRequest.getDroppedSize().add(droppedSize));
+ swapMode = false;
+ incrementSwapQueueSize(-droppedSize.getObjectCount(), -droppedSize.getByteCount());
+ logger.debug("For DropFlowFileRequest {}, dropped {} from Swap Queue", requestIdentifier, droppedSize);
+
+ final int swapFileCount = swapLocations.size();
+ final Iterator<String> swapLocationItr = swapLocations.iterator();
+ while (swapLocationItr.hasNext()) {
+ final String swapLocation = swapLocationItr.next();
+
+ List<FlowFileRecord> swappedIn = null;
+ try {
+ swappedIn = swapManager.swapIn(swapLocation, StandardFlowFileQueue.this);
+ droppedSize = drop(swappedIn, requestor);
+ } catch (final IOException ioe) {
+ logger.error("Failed to swap in FlowFiles from Swap File {} in order to drop the FlowFiles for Connection {} due to {}",
+ swapLocation, StandardFlowFileQueue.this.getIdentifier(), ioe.toString());
+ logger.error("", ioe);
+
+ dropRequest.setState(DropFlowFileState.FAILURE, "Failed to swap in FlowFiles from Swap File " + swapLocation + " due to " + ioe.toString());
+ if (swappedIn != null) {
+ activeQueue.addAll(swappedIn); // ensure that we don't lose the FlowFiles from our queue.
+ }
+
+ return;
+ }
+
+ dropRequest.setDroppedSize(dropRequest.getDroppedSize().add(droppedSize));
+ incrementSwapQueueSize(-droppedSize.getObjectCount(), -droppedSize.getByteCount());
+
+ dropRequest.setCurrentSize(getQueueSize());
+ swapLocationItr.remove();
+ logger.debug("For DropFlowFileRequest {}, dropped {} for Swap File {}", requestIdentifier, droppedSize, swapLocation);
+ }
+
+ logger.debug("Dropped FlowFiles from {} Swap Files", swapFileCount);
+ logger.info("Successfully dropped {} FlowFiles ({} bytes) from Connection with ID {} on behalf of {}",
+ dropRequest.getDroppedSize().getObjectCount(), dropRequest.getDroppedSize().getByteCount(), StandardFlowFileQueue.this.getIdentifier(), requestor);
+ dropRequest.setState(DropFlowFileState.COMPLETE);
+ } catch (final Exception e) {
+ logger.error("Failed to drop FlowFiles from Connection with ID {} due to {}", StandardFlowFileQueue.this.getIdentifier(), e.toString());
+ logger.error("", e);
+ dropRequest.setState(DropFlowFileState.FAILURE, "Failed to drop FlowFiles due to " + e.toString());
+ }
+ } finally {
+ writeLock.unlock("Drop FlowFiles");
+ }
+ }
+ }, "Drop FlowFiles for Connection " + getIdentifier());
+ t.setDaemon(true);
+ t.start();
+
+ dropRequest.setExecutionThread(t);
+ dropRequestMap.put(requestIdentifier, dropRequest);
+
+ return dropRequest;
+ }
+
+ private QueueSize drop(final List<FlowFileRecord> flowFiles, final String requestor) throws IOException {
+ // Create a Provenance Event and a FlowFile Repository record for each FlowFile
+ final List<ProvenanceEventRecord> provenanceEvents = new ArrayList<>(flowFiles.size());
+ final List<RepositoryRecord> flowFileRepoRecords = new ArrayList<>(flowFiles.size());
+ for (final FlowFileRecord flowFile : flowFiles) {
+ provenanceEvents.add(createDropEvent(flowFile, requestor));
+ flowFileRepoRecords.add(createDeleteRepositoryRecord(flowFile));
+ }
+
+ long dropContentSize = 0L;
+ for (final FlowFileRecord flowFile : flowFiles) {
+ dropContentSize += flowFile.getSize();
+ final ContentClaim contentClaim = flowFile.getContentClaim();
+ if (contentClaim == null) {
+ continue;
+ }
+
+ final ResourceClaim resourceClaim = contentClaim.getResourceClaim();
+ if (resourceClaim == null) {
+ continue;
+ }
+
+ resourceClaimManager.decrementClaimantCount(resourceClaim);
+ }
+
+ provRepository.registerEvents(provenanceEvents);
+ flowFileRepository.updateRepository(flowFileRepoRecords);
+ return new QueueSize(flowFiles.size(), dropContentSize);
+ }
+
+ private ProvenanceEventRecord createDropEvent(final FlowFileRecord flowFile, final String requestor) {
+ final ProvenanceEventBuilder builder = provRepository.eventBuilder();
+ builder.fromFlowFile(flowFile);
+ builder.setEventType(ProvenanceEventType.DROP);
+ builder.setLineageStartDate(flowFile.getLineageStartDate());
+ builder.setComponentId(getIdentifier());
+ builder.setComponentType("Connection");
+ builder.setDetails("FlowFile manually dropped; request made by " + requestor);
+ return builder.build();
+ }
+
+ private RepositoryRecord createDeleteRepositoryRecord(final FlowFileRecord flowFile) {
+ return new RepositoryRecord() {
+ @Override
+ public FlowFileQueue getDestination() {
+ return null;
+ }
+
+ @Override
+ public FlowFileQueue getOriginalQueue() {
+ return StandardFlowFileQueue.this;
+ }
+
+ @Override
+ public RepositoryRecordType getType() {
+ return RepositoryRecordType.DELETE;
+ }
+
+ @Override
+ public ContentClaim getCurrentClaim() {
+ return flowFile.getContentClaim();
+ }
+
+ @Override
+ public ContentClaim getOriginalClaim() {
+ return flowFile.getContentClaim();
+ }
+
+ @Override
+ public long getCurrentClaimOffset() {
+ return flowFile.getContentClaimOffset();
+ }
+
+ @Override
+ public FlowFileRecord getCurrent() {
+ return flowFile;
+ }
+
+ @Override
+ public boolean isAttributesChanged() {
+ return false;
+ }
+
+ @Override
+ public boolean isMarkedForAbort() {
+ return false;
+ }
+
+ @Override
+ public String getSwapLocation() {
+ return null;
+ }
+ };
+ }
+
+
+ @Override
+ public boolean cancelDropFlowFileRequest(final String requestIdentifier) {
+ final DropFlowFileRequest request = dropRequestMap.remove(requestIdentifier);
+ if (request == null) {
+ return false;
+ }
+
+ final boolean successful = request.cancel();
+ return successful;
+ }
+
+ @Override
+ public DropFlowFileStatus getDropFlowFileStatus(final String requestIdentifier) {
+ return dropRequestMap.get(requestIdentifier);
+ }
+
+ /**
+ * Lock the queue so that other threads are unable to interact with the
+ * queue
+ */
+ public void lock() {
+ writeLock.lock();
+ }
+
+ /**
+ * Unlock the queue
+ */
+ public void unlock() {
+ writeLock.unlock("external unlock");
+ }
+
+ @Override
+ public QueueSize getUnacknowledgedQueueSize() {
+ return size.get().unacknowledgedQueueSize();
+ }
+
+
+ private void incrementActiveQueueSize(final int count, final long bytes) {
+ boolean updated = false;
+ while (!updated) {
+ final FlowFileQueueSize original = size.get();
+ final FlowFileQueueSize newSize = new FlowFileQueueSize(original.activeQueueCount + count, original.activeQueueBytes + bytes,
+ original.swappedCount, original.swappedBytes, original.unacknowledgedCount, original.unacknowledgedBytes);
+ updated = size.compareAndSet(original, newSize);
+ }
+ }
+
+ private void incrementSwapQueueSize(final int count, final long bytes) {
+ boolean updated = false;
+ while (!updated) {
+ final FlowFileQueueSize original = size.get();
+ final FlowFileQueueSize newSize = new FlowFileQueueSize(original.activeQueueCount, original.activeQueueBytes,
+ original.swappedCount + count, original.swappedBytes + bytes, original.unacknowledgedCount, original.unacknowledgedBytes);
+ updated = size.compareAndSet(original, newSize);
+ }
+ }
+
+ private void incrementUnacknowledgedQueueSize(final int count, final long bytes) {
+ boolean updated = false;
+ while (!updated) {
+ final FlowFileQueueSize original = size.get();
+ final FlowFileQueueSize newSize = new FlowFileQueueSize(original.activeQueueCount, original.activeQueueBytes,
+ original.swappedCount, original.swappedBytes, original.unacknowledgedCount + count, original.unacknowledgedBytes + bytes);
+ updated = size.compareAndSet(original, newSize);
+ }
+ }
+
+
+ private static class FlowFileQueueSize {
+ private final int activeQueueCount;
+ private final long activeQueueBytes;
+ private final int swappedCount;
+ private final long swappedBytes;
+ private final int unacknowledgedCount;
+ private final long unacknowledgedBytes;
+
+ public FlowFileQueueSize(final int activeQueueCount, final long activeQueueBytes, final int swappedCount, final long swappedBytes,
+ final int unacknowledgedCount, final long unacknowledgedBytes) {
+ this.activeQueueCount = activeQueueCount;
+ this.activeQueueBytes = activeQueueBytes;
+ this.swappedCount = swappedCount;
+ this.swappedBytes = swappedBytes;
+ this.unacknowledgedCount = unacknowledgedCount;
+ this.unacknowledgedBytes = unacknowledgedBytes;
+ }
+
+ public boolean isEmpty() {
+ return activeQueueCount == 0 && swappedCount == 0 && unacknowledgedCount == 0;
+ }
+
+ public QueueSize toQueueSize() {
+ return new QueueSize(activeQueueCount + swappedCount + unacknowledgedCount, activeQueueBytes + swappedBytes + unacknowledgedBytes);
+ }
+
+ public QueueSize activeQueueSize() {
+ return new QueueSize(activeQueueCount, activeQueueBytes);
+ }
+
+ public QueueSize unacknowledgedQueueSize() {
+ return new QueueSize(unacknowledgedCount, unacknowledgedBytes);
+ }
+
+ public QueueSize swapQueueSize() {
+ return new QueueSize(swappedCount, swappedBytes);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/77f7d752/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestStandardFlowFileQueue.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestStandardFlowFileQueue.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestStandardFlowFileQueue.java
new file mode 100644
index 0000000..3789ea5
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestStandardFlowFileQueue.java
@@ -0,0 +1,356 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.controller;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.nifi.connectable.Connectable;
+import org.apache.nifi.connectable.Connection;
+import org.apache.nifi.controller.queue.DropFlowFileState;
+import org.apache.nifi.controller.queue.DropFlowFileStatus;
+import org.apache.nifi.controller.queue.FlowFileQueue;
+import org.apache.nifi.controller.queue.QueueSize;
+import org.apache.nifi.controller.repository.FlowFileRecord;
+import org.apache.nifi.controller.repository.FlowFileRepository;
+import org.apache.nifi.controller.repository.FlowFileSwapManager;
+import org.apache.nifi.controller.repository.SwapManagerInitializationContext;
+import org.apache.nifi.controller.repository.claim.ContentClaim;
+import org.apache.nifi.controller.repository.claim.ResourceClaimManager;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.FlowFilePrioritizer;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.provenance.ProvenanceEventRepository;
+import org.apache.nifi.provenance.StandardProvenanceEventRecord;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+public class TestStandardFlowFileQueue {
+ private TestSwapManager swapManager = null;
+ private StandardFlowFileQueue queue = null;
+
+ @Before
+ public void setup() {
+ final Connection connection = Mockito.mock(Connection.class);
+ Mockito.when(connection.getSource()).thenReturn(Mockito.mock(Connectable.class));
+ Mockito.when(connection.getDestination()).thenReturn(Mockito.mock(Connectable.class));
+
+ final ProcessScheduler scheduler = Mockito.mock(ProcessScheduler.class);
+ swapManager = new TestSwapManager();
+
+ final FlowFileRepository flowFileRepo = Mockito.mock(FlowFileRepository.class);
+ final ProvenanceEventRepository provRepo = Mockito.mock(ProvenanceEventRepository.class);
+ final ResourceClaimManager claimManager = Mockito.mock(ResourceClaimManager.class);
+
+ Mockito.when(provRepo.eventBuilder()).thenReturn(new StandardProvenanceEventRecord.Builder());
+
+ queue = new StandardFlowFileQueue("id", connection, flowFileRepo, provRepo, claimManager, scheduler, swapManager, null, 10000);
+ TestFlowFile.idGenerator.set(0L);
+ }
+
+
+ @Test
+ public void testSwapOutOccurs() {
+ for (int i = 0; i < 10000; i++) {
+ queue.put(new TestFlowFile());
+ assertEquals(0, swapManager.swapOutCalledCount);
+ assertEquals(i + 1, queue.size().getObjectCount());
+ assertEquals(i + 1, queue.size().getByteCount());
+ }
+
+ for (int i = 0; i < 9999; i++) {
+ queue.put(new TestFlowFile());
+ assertEquals(0, swapManager.swapOutCalledCount);
+ assertEquals(i + 10001, queue.size().getObjectCount());
+ assertEquals(i + 10001, queue.size().getByteCount());
+ }
+
+ queue.put(new TestFlowFile(1000));
+ assertEquals(1, swapManager.swapOutCalledCount);
+ assertEquals(20000, queue.size().getObjectCount());
+ assertEquals(20999, queue.size().getByteCount());
+
+ assertEquals(10000, queue.getActiveQueueSize().getObjectCount());
+ }
+
+ @Test
+ public void testLowestPrioritySwappedOutFirst() {
+ final List<FlowFilePrioritizer> prioritizers = new ArrayList<>();
+ prioritizers.add(new FlowFileSizePrioritizer());
+ queue.setPriorities(prioritizers);
+
+ long maxSize = 20000;
+ for (int i = 1; i <= 20000; i++) {
+ queue.put(new TestFlowFile(maxSize - i));
+ }
+
+ assertEquals(1, swapManager.swapOutCalledCount);
+ assertEquals(20000, queue.size().getObjectCount());
+
+ assertEquals(10000, queue.getActiveQueueSize().getObjectCount());
+ final List<FlowFileRecord> flowFiles = queue.poll(Integer.MAX_VALUE, new HashSet<FlowFileRecord>());
+ assertEquals(10000, flowFiles.size());
+ for (int i = 0; i < 10000; i++) {
+ assertEquals(i, flowFiles.get(i).getSize());
+ }
+ }
+
+ @Test
+ public void testSwapIn() {
+ for (int i = 1; i <= 20000; i++) {
+ queue.put(new TestFlowFile());
+ }
+
+ assertEquals(1, swapManager.swappedOut.size());
+ queue.put(new TestFlowFile());
+ assertEquals(1, swapManager.swappedOut.size());
+
+ final Set<FlowFileRecord> exp = new HashSet<>();
+ for (int i = 0; i < 9999; i++) {
+ assertNotNull(queue.poll(exp));
+ }
+
+ assertEquals(0, swapManager.swapInCalledCount);
+ assertEquals(1, queue.getActiveQueueSize().getObjectCount());
+ assertNotNull(queue.poll(exp));
+
+ assertEquals(0, swapManager.swapInCalledCount);
+ assertEquals(0, queue.getActiveQueueSize().getObjectCount());
+
+ assertEquals(1, swapManager.swapOutCalledCount);
+
+ assertNotNull(queue.poll(exp)); // this should trigger a swap-in of 10,000 records, and then pull 1 off the top.
+ assertEquals(1, swapManager.swapInCalledCount);
+ assertEquals(9999, queue.getActiveQueueSize().getObjectCount());
+
+ assertTrue(swapManager.swappedOut.isEmpty());
+
+ queue.poll(exp);
+ }
+
+ @Test
+ public void testDropSwappedFlowFiles() {
+ for (int i = 1; i <= 210000; i++) {
+ queue.put(new TestFlowFile());
+ }
+
+ assertEquals(20, swapManager.swappedOut.size());
+ final DropFlowFileStatus status = queue.dropFlowFiles("1", "Unit Test");
+ while (status.getState() != DropFlowFileState.COMPLETE) {
+ final QueueSize queueSize = queue.size();
+ System.out.println(queueSize);
+ try {
+ Thread.sleep(1000L);
+ } catch (final Exception e) {
+ }
+ }
+
+ System.out.println(queue.size());
+ assertEquals(0, queue.size().getObjectCount());
+ assertEquals(0, queue.size().getByteCount());
+ assertEquals(0, swapManager.swappedOut.size());
+ assertEquals(20, swapManager.swapInCalledCount);
+ }
+
+ private class TestSwapManager implements FlowFileSwapManager {
+ private final Map<String, List<FlowFileRecord>> swappedOut = new HashMap<>();
+ int swapOutCalledCount = 0;
+ int swapInCalledCount = 0;
+
+ @Override
+ public void initialize(final SwapManagerInitializationContext initializationContext) {
+
+ }
+
+ @Override
+ public String swapOut(List<FlowFileRecord> flowFiles, FlowFileQueue flowFileQueue) throws IOException {
+ swapOutCalledCount++;
+ final String location = UUID.randomUUID().toString();
+ swappedOut.put(location, new ArrayList<FlowFileRecord>(flowFiles));
+ return location;
+ }
+
+ @Override
+ public List<FlowFileRecord> peek(String swapLocation, final FlowFileQueue flowFileQueue) throws IOException {
+ return new ArrayList<FlowFileRecord>(swappedOut.get(swapLocation));
+ }
+
+ @Override
+ public List<FlowFileRecord> swapIn(String swapLocation, FlowFileQueue flowFileQueue) throws IOException {
+ swapInCalledCount++;
+ return swappedOut.remove(swapLocation);
+ }
+
+ @Override
+ public List<String> recoverSwapLocations(FlowFileQueue flowFileQueue) throws IOException {
+ return new ArrayList<String>(swappedOut.keySet());
+ }
+
+ @Override
+ public QueueSize getSwapSize(String swapLocation) throws IOException {
+ final List<FlowFileRecord> flowFiles = swappedOut.get(swapLocation);
+ if (flowFiles == null) {
+ return new QueueSize(0, 0L);
+ }
+
+ int count = 0;
+ long size = 0L;
+ for (final FlowFileRecord flowFile : flowFiles) {
+ count++;
+ size += flowFile.getSize();
+ }
+
+ return new QueueSize(count, size);
+ }
+
+ @Override
+ public Long getMaxRecordId(String swapLocation) throws IOException {
+ final List<FlowFileRecord> flowFiles = swappedOut.get(swapLocation);
+ if (flowFiles == null) {
+ return null;
+ }
+
+ Long max = null;
+ for (final FlowFileRecord flowFile : flowFiles) {
+ if (max == null || flowFile.getId() > max) {
+ max = flowFile.getId();
+ }
+ }
+
+ return max;
+ }
+
+ @Override
+ public void purge() {
+ swappedOut.clear();
+ }
+ }
+
+
+ private static class TestFlowFile implements FlowFileRecord {
+ private static final AtomicLong idGenerator = new AtomicLong(0L);
+
+ private final long id = idGenerator.getAndIncrement();
+ private final long entryDate = System.currentTimeMillis();
+ private final Map<String, String> attributes;
+ private final long size;
+
+ public TestFlowFile() {
+ this(1L);
+ }
+
+ public TestFlowFile(final long size) {
+ this(new HashMap<String, String>(), size);
+ }
+
+ public TestFlowFile(final Map<String, String> attributes, final long size) {
+ this.attributes = attributes;
+ this.size = size;
+
+ if (!attributes.containsKey(CoreAttributes.UUID.key())) {
+ attributes.put(CoreAttributes.UUID.key(), UUID.randomUUID().toString());
+ }
+ }
+
+
+ @Override
+ public long getId() {
+ return id;
+ }
+
+ @Override
+ public long getEntryDate() {
+ return entryDate;
+ }
+
+ @Override
+ public long getLineageStartDate() {
+ return entryDate;
+ }
+
+ @Override
+ public Long getLastQueueDate() {
+ return null;
+ }
+
+ @Override
+ public Set<String> getLineageIdentifiers() {
+ return Collections.emptySet();
+ }
+
+ @Override
+ public boolean isPenalized() {
+ return false;
+ }
+
+ @Override
+ public String getAttribute(String key) {
+ return attributes.get(key);
+ }
+
+ @Override
+ public long getSize() {
+ return size;
+ }
+
+ @Override
+ public Map<String, String> getAttributes() {
+ return Collections.unmodifiableMap(attributes);
+ }
+
+ @Override
+ public int compareTo(final FlowFile o) {
+ return Long.compare(id, o.getId());
+ }
+
+ @Override
+ public long getPenaltyExpirationMillis() {
+ return 0;
+ }
+
+ @Override
+ public ContentClaim getContentClaim() {
+ return null;
+ }
+
+ @Override
+ public long getContentClaimOffset() {
+ return 0;
+ }
+ }
+
+ private static class FlowFileSizePrioritizer implements FlowFilePrioritizer {
+ @Override
+ public int compare(final FlowFile o1, final FlowFile o2) {
+ return Long.compare(o1.getSize(), o2.getSize());
+ }
+ }
+}
[06/30] nifi git commit: Merge branch 'NIFI-730' of
https://git-wip-us.apache.org/repos/asf/nifi into NIFI-730
Posted by ma...@apache.org.
Merge branch 'NIFI-730' of https://git-wip-us.apache.org/repos/asf/nifi into NIFI-730
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/9be37914
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/9be37914
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/9be37914
Branch: refs/heads/master
Commit: 9be37914ddb9c8c017cc4d6b3209340a19a7cb8d
Parents: 49a781d e0ac7cd
Author: Mark Payne <ma...@hotmail.com>
Authored: Tue Oct 13 10:03:31 2015 -0400
Committer: Mark Payne <ma...@hotmail.com>
Committed: Tue Oct 13 10:03:31 2015 -0400
----------------------------------------------------------------------
.../apache/nifi/web/api/dto/DropRequestDTO.java | 129 +++++++++++
.../nifi/web/api/entity/DropRequestEntity.java | 44 ++++
.../org/apache/nifi/web/NiFiServiceFacade.java | 25 +++
.../nifi/web/StandardNiFiServiceFacade.java | 24 ++
.../apache/nifi/web/api/ConnectionResource.java | 217 ++++++++++++++++++-
.../org/apache/nifi/web/dao/ConnectionDAO.java | 22 ++
.../web/dao/impl/StandardConnectionDAO.java | 15 ++
.../src/main/webapp/js/nf/canvas/nf-actions.js | 58 +++++
.../main/webapp/js/nf/canvas/nf-context-menu.js | 10 +
9 files changed, 542 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[07/30] nifi git commit: NIFI-730: Fixed checkstyle violations
Posted by ma...@apache.org.
NIFI-730: Fixed checkstyle violations
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/ad6af95d
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/ad6af95d
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/ad6af95d
Branch: refs/heads/master
Commit: ad6af95d079c277bf2a7bc63277f14a91fddebf7
Parents: 9be3791
Author: Mark Payne <ma...@hotmail.com>
Authored: Tue Oct 13 10:09:21 2015 -0400
Committer: Mark Payne <ma...@hotmail.com>
Committed: Tue Oct 13 10:09:21 2015 -0400
----------------------------------------------------------------------
.../org/apache/nifi/controller/queue/DropFlowFileState.java | 6 +++---
.../nifi-azure-bundle/nifi-azure-processors/.gitignore | 1 +
2 files changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/ad6af95d/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileState.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileState.java b/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileState.java
index e412b80..6462dfd 100644
--- a/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileState.java
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileState.java
@@ -27,13 +27,13 @@ public enum DropFlowFileState {
COMPLETE("Completed Successfully"),
FAILURE("Failed"),
CANCELED("Cancelled by User");
-
+
private final String description;
-
+
private DropFlowFileState(final String description) {
this.description = description;
}
-
+
@Override
public String toString() {
return description;
http://git-wip-us.apache.org/repos/asf/nifi/blob/ad6af95d/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/.gitignore
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/.gitignore b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/.gitignore
new file mode 100644
index 0000000..ae3c172
--- /dev/null
+++ b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/.gitignore
@@ -0,0 +1 @@
+/bin/
[12/30] nifi git commit: NIFI-730: Added error messages if we fail to
drop FlowFiles from queue
Posted by ma...@apache.org.
NIFI-730: Added error messages if we fail to drop FlowFiles from queue
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/afb76afc
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/afb76afc
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/afb76afc
Branch: refs/heads/master
Commit: afb76afcd0fd7d0c144a37621fdabc181bd42307
Parents: 72ff2a2
Author: Mark Payne <ma...@hotmail.com>
Authored: Tue Oct 13 15:57:18 2015 -0400
Committer: Mark Payne <ma...@hotmail.com>
Committed: Tue Oct 13 15:57:18 2015 -0400
----------------------------------------------------------------------
.../controller/queue/DropFlowFileStatus.java | 5 +++
.../nifi/controller/DropFlowFileRequest.java | 11 +++++
.../nifi/controller/StandardFlowFileQueue.java | 45 ++++++++++++++++----
3 files changed, 53 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/afb76afc/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileStatus.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileStatus.java b/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileStatus.java
index 7d5b9c2..737fbe3 100644
--- a/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileStatus.java
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileStatus.java
@@ -70,4 +70,9 @@ public interface DropFlowFileStatus {
* @return the current state of the operation
*/
DropFlowFileState getState();
+
+ /**
+ * @return the reason that the state is set to a Failure state, or <code>null</code> if the state is not {@link DropFlowFileState#FAILURE}.
+ */
+ String getFailureReason();
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/afb76afc/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/DropFlowFileRequest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/DropFlowFileRequest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/DropFlowFileRequest.java
index 4104308..189fe7d 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/DropFlowFileRequest.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/DropFlowFileRequest.java
@@ -30,6 +30,7 @@ public class DropFlowFileRequest implements DropFlowFileStatus {
private volatile QueueSize droppedSize = new QueueSize(0, 0L);
private volatile long lastUpdated = System.currentTimeMillis();
private volatile Thread executionThread;
+ private volatile String failureReason;
private DropFlowFileState state = DropFlowFileState.WAITING_FOR_LOCK;
@@ -85,8 +86,18 @@ public class DropFlowFileRequest implements DropFlowFileStatus {
return lastUpdated;
}
+ @Override
+ public String getFailureReason() {
+ return failureReason;
+ }
+
synchronized void setState(final DropFlowFileState state) {
+ setState(state, null);
+ }
+
+ synchronized void setState(final DropFlowFileState state, final String explanation) {
this.state = state;
+ this.failureReason = explanation;
this.lastUpdated = System.currentTimeMillis();
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/afb76afc/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
index 82c1c7e..5b137f7 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
@@ -928,14 +928,34 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
try {
final List<FlowFileRecord> activeQueueRecords = new ArrayList<>(activeQueue);
- QueueSize droppedSize = drop(activeQueueRecords, requestor);
+
+ QueueSize droppedSize;
+ try {
+ droppedSize = drop(activeQueueRecords, requestor);
+ } catch (final IOException ioe) {
+ logger.error("Failed to drop the FlowFiles from queue {} due to {}", StandardFlowFileQueue.this.getIdentifier(), ioe.toString());
+ logger.error("", ioe);
+
+ dropRequest.setState(DropFlowFileState.FAILURE, "Failed to drop FlowFiles due to " + ioe.toString());
+ return;
+ }
+
activeQueue.clear();
activeQueueContentSize = 0;
activeQueueSizeRef.set(0);
dropRequest.setCurrentSize(getQueueSize());
dropRequest.setDroppedSize(dropRequest.getDroppedSize().add(droppedSize));
- droppedSize = drop(swapQueue, requestor);
+ try {
+ droppedSize = drop(swapQueue, requestor);
+ } catch (final IOException ioe) {
+ logger.error("Failed to drop the FlowFiles from queue {} due to {}", StandardFlowFileQueue.this.getIdentifier(), ioe.toString());
+ logger.error("", ioe);
+
+ dropRequest.setState(DropFlowFileState.FAILURE, "Failed to drop FlowFiles due to " + ioe.toString());
+ return;
+ }
+
swapQueue.clear();
dropRequest.setCurrentSize(getQueueSize());
dropRequest.setDroppedSize(dropRequest.getDroppedSize().add(droppedSize));
@@ -946,12 +966,22 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
final Iterator<String> swapLocationItr = swapLocations.iterator();
while (swapLocationItr.hasNext()) {
final String swapLocation = swapLocationItr.next();
- final List<FlowFileRecord> swappedIn = swapManager.swapIn(swapLocation, StandardFlowFileQueue.this);
+
+ List<FlowFileRecord> swappedIn = null;
try {
+ swappedIn = swapManager.swapIn(swapLocation, StandardFlowFileQueue.this);
droppedSize = drop(swappedIn, requestor);
- } catch (final Exception e) {
- activeQueue.addAll(swappedIn); // ensure that we don't lose the FlowFiles from our queue.
- throw e;
+ } catch (final IOException ioe) {
+ logger.error("Failed to swap in FlowFiles from Swap File {} in order to drop the FlowFiles for Connection {} due to {}",
+ swapLocation, StandardFlowFileQueue.this.getIdentifier(), ioe.toString());
+ logger.error("", ioe);
+
+ dropRequest.setState(DropFlowFileState.FAILURE, "Failed to swap in FlowFiles from Swap File " + swapLocation + " due to " + ioe.toString());
+ if (swappedIn != null) {
+ activeQueue.addAll(swappedIn); // ensure that we don't lose the FlowFiles from our queue.
+ }
+
+ return;
}
dropRequest.setDroppedSize(dropRequest.getDroppedSize().add(droppedSize));
@@ -963,8 +993,7 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
dropRequest.setState(DropFlowFileState.COMPLETE);
} catch (final Exception e) {
- // TODO: Handle adequately
- dropRequest.setState(DropFlowFileState.FAILURE);
+ dropRequest.setState(DropFlowFileState.FAILURE, "Failed to drop FlowFiles due to " + e.toString());
}
} finally {
writeLock.unlock("Drop FlowFiles");
[16/30] nifi git commit: NIFI-730: Added additional testing logic to
unit tests
Posted by ma...@apache.org.
NIFI-730: Added additional testing logic to unit tests
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/5867193b
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/5867193b
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/5867193b
Branch: refs/heads/master
Commit: 5867193bc131962d05985f405ea199840a99feb9
Parents: 0af1aca
Author: Mark Payne <ma...@hotmail.com>
Authored: Wed Oct 14 09:56:52 2015 -0400
Committer: Mark Payne <ma...@hotmail.com>
Committed: Wed Oct 14 09:56:52 2015 -0400
----------------------------------------------------------------------
.../apache/nifi/controller/TestStandardFlowFileQueue.java | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/5867193b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestStandardFlowFileQueue.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestStandardFlowFileQueue.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestStandardFlowFileQueue.java
index 4b67d91..7ef5fc8 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestStandardFlowFileQueue.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestStandardFlowFileQueue.java
@@ -135,7 +135,14 @@ public class TestStandardFlowFileQueue {
final Set<FlowFileRecord> exp = new HashSet<>();
for (int i = 0; i < 9999; i++) {
- assertNotNull(queue.poll(exp));
+ final FlowFileRecord flowFile = queue.poll(exp);
+ assertNotNull(flowFile);
+ assertEquals(1, queue.getUnacknowledgedQueueSize().getObjectCount());
+ assertEquals(1, queue.getUnacknowledgedQueueSize().getByteCount());
+
+ queue.acknowledge(Collections.singleton(flowFile));
+ assertEquals(0, queue.getUnacknowledgedQueueSize().getObjectCount());
+ assertEquals(0, queue.getUnacknowledgedQueueSize().getByteCount());
}
assertEquals(0, swapManager.swapInCalledCount);
[27/30] nifi git commit: NIFI-730: Ensure that we always populate
queue counts when initiating a Drop FlowFile request
Posted by ma...@apache.org.
NIFI-730: Ensure that we always populate queue counts when initiating a Drop FlowFile request
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/f5727cfb
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/f5727cfb
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/f5727cfb
Branch: refs/heads/master
Commit: f5727cfb0fe73b7226fe9eca96fd594f8d423ffa
Parents: edf238e
Author: Mark Payne <ma...@hotmail.com>
Authored: Mon Nov 2 08:36:50 2015 -0500
Committer: Mark Payne <ma...@hotmail.com>
Committed: Mon Nov 2 08:36:50 2015 -0500
----------------------------------------------------------------------
.../apache/nifi/controller/StandardFlowFileQueue.java | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/f5727cfb/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
index 0f6d258..6b6bb57 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
@@ -923,7 +923,15 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
}
final DropFlowFileRequest dropRequest = new DropFlowFileRequest(requestIdentifier);
- dropRequest.setCurrentSize(size());
+ final QueueSize originalSize = getQueueSize();
+ dropRequest.setCurrentSize(originalSize);
+ dropRequest.setOriginalSize(originalSize);
+ if (originalSize.getObjectCount() == 0) {
+ dropRequest.setDroppedSize(originalSize);
+ dropRequest.setState(DropFlowFileState.COMPLETE);
+ return dropRequest;
+ }
+
final Thread t = new Thread(new Runnable() {
@Override
public void run() {
@@ -931,7 +939,6 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
try {
dropRequest.setState(DropFlowFileState.DROPPING_FLOWFILES);
logger.debug("For DropFlowFileRequest {}, original size is {}", requestIdentifier, getQueueSize());
- dropRequest.setOriginalSize(getQueueSize());
try {
final List<FlowFileRecord> activeQueueRecords = new ArrayList<>(activeQueue);
[14/30] nifi git commit: NIFI-730: bug fixes and code cleanup for
swap manager and flowfile queue
Posted by ma...@apache.org.
NIFI-730: bug fixes and code cleanup for swap manager and flowfile queue
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/77f7d752
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/77f7d752
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/77f7d752
Branch: refs/heads/master
Commit: 77f7d7524cb8b07ed2976088f0e57d99233c8327
Parents: afb76af
Author: Mark Payne <ma...@hotmail.com>
Authored: Wed Oct 14 09:14:15 2015 -0400
Committer: Mark Payne <ma...@hotmail.com>
Committed: Wed Oct 14 09:14:15 2015 -0400
----------------------------------------------------------------------
.../repository/FlowFileSwapManager.java | 10 -
.../nifi/controller/DropFlowFileRequest.java | 2 +-
.../nifi/controller/StandardFlowFileQueue.java | 1153 ----------------
.../controller/TestStandardFlowFileQueue.java | 330 -----
.../nifi/controller/FileSystemSwapManager.java | 5 -
.../nifi/controller/StandardFlowFileQueue.java | 1246 ++++++++++++++++++
.../controller/TestStandardFlowFileQueue.java | 356 +++++
7 files changed, 1603 insertions(+), 1499 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/77f7d752/nifi-api/src/main/java/org/apache/nifi/controller/repository/FlowFileSwapManager.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/repository/FlowFileSwapManager.java b/nifi-api/src/main/java/org/apache/nifi/controller/repository/FlowFileSwapManager.java
index a70d287..3e341f8 100644
--- a/nifi-api/src/main/java/org/apache/nifi/controller/repository/FlowFileSwapManager.java
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/repository/FlowFileSwapManager.java
@@ -41,16 +41,6 @@ public interface FlowFileSwapManager {
void initialize(SwapManagerInitializationContext initializationContext);
/**
- * Drops all FlowFiles that are swapped out at the given location. This will update the Provenance
- * Repository as well as the FlowFile Repository and
- *
- * @param swapLocation the location of the swap file to drop
- * @param flowFileQueue the queue to which the FlowFiles belong
- * @param user the user that initiated the request
- */
- void dropSwappedFlowFiles(String swapLocation, FlowFileQueue flowFileQueue, String user) throws IOException;
-
- /**
* Swaps out the given FlowFiles that belong to the queue with the given identifier.
*
* @param flowFiles the FlowFiles to swap out to external storage
http://git-wip-us.apache.org/repos/asf/nifi/blob/77f7d752/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/DropFlowFileRequest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/DropFlowFileRequest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/DropFlowFileRequest.java
index 189fe7d..58695c2 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/DropFlowFileRequest.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/DropFlowFileRequest.java
@@ -64,7 +64,7 @@ public class DropFlowFileRequest implements DropFlowFileStatus {
}
void setCurrentSize(final QueueSize queueSize) {
- this.currentSize = currentSize;
+ this.currentSize = queueSize;
}
@Override
http://git-wip-us.apache.org/repos/asf/nifi/blob/77f7d752/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
deleted file mode 100644
index 5b137f7..0000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
+++ /dev/null
@@ -1,1153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.controller;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.PriorityQueue;
-import java.util.Queue;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import org.apache.nifi.connectable.Connection;
-import org.apache.nifi.controller.queue.DropFlowFileState;
-import org.apache.nifi.controller.queue.DropFlowFileStatus;
-import org.apache.nifi.controller.queue.FlowFileQueue;
-import org.apache.nifi.controller.queue.QueueSize;
-import org.apache.nifi.controller.repository.FlowFileRecord;
-import org.apache.nifi.controller.repository.FlowFileRepository;
-import org.apache.nifi.controller.repository.FlowFileSwapManager;
-import org.apache.nifi.controller.repository.RepositoryRecord;
-import org.apache.nifi.controller.repository.RepositoryRecordType;
-import org.apache.nifi.controller.repository.claim.ContentClaim;
-import org.apache.nifi.controller.repository.claim.ResourceClaim;
-import org.apache.nifi.controller.repository.claim.ResourceClaimManager;
-import org.apache.nifi.events.EventReporter;
-import org.apache.nifi.flowfile.FlowFile;
-import org.apache.nifi.flowfile.FlowFilePrioritizer;
-import org.apache.nifi.processor.DataUnit;
-import org.apache.nifi.processor.FlowFileFilter;
-import org.apache.nifi.processor.FlowFileFilter.FlowFileFilterResult;
-import org.apache.nifi.provenance.ProvenanceEventBuilder;
-import org.apache.nifi.provenance.ProvenanceEventRecord;
-import org.apache.nifi.provenance.ProvenanceEventRepository;
-import org.apache.nifi.provenance.ProvenanceEventType;
-import org.apache.nifi.reporting.Severity;
-import org.apache.nifi.scheduling.SchedulingStrategy;
-import org.apache.nifi.util.FormatUtils;
-import org.apache.nifi.util.concurrency.TimedLock;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * A FlowFileQueue is used to queue FlowFile objects that are awaiting further
- * processing. Must be thread safe.
- *
- */
-public final class StandardFlowFileQueue implements FlowFileQueue {
-
- public static final int MAX_EXPIRED_RECORDS_PER_ITERATION = 100000;
- public static final int SWAP_RECORD_POLL_SIZE = 10000;
-
- private static final Logger logger = LoggerFactory.getLogger(StandardFlowFileQueue.class);
-
- private PriorityQueue<FlowFileRecord> activeQueue = null;
- private long activeQueueContentSize = 0L;
- private ArrayList<FlowFileRecord> swapQueue = null;
-
- private int swappedRecordCount = 0;
- private long swappedContentSize = 0L;
- private String maximumQueueDataSize;
- private long maximumQueueByteCount;
- private boolean swapMode = false;
- private long maximumQueueObjectCount;
-
- private final EventReporter eventReporter;
- private final AtomicLong flowFileExpirationMillis;
- private final Connection connection;
- private final AtomicReference<String> flowFileExpirationPeriod;
- private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true);
- private final List<FlowFilePrioritizer> priorities;
- private final int swapThreshold;
- private final FlowFileSwapManager swapManager;
- private final List<String> swapLocations = new ArrayList<>();
- private final TimedLock readLock;
- private final TimedLock writeLock;
- private final String identifier;
- private final FlowFileRepository flowFileRepository;
- private final ProvenanceEventRepository provRepository;
- private final ResourceClaimManager resourceClaimManager;
-
- private final AtomicBoolean queueFullRef = new AtomicBoolean(false);
- private final AtomicInteger activeQueueSizeRef = new AtomicInteger(0);
- private final AtomicReference<QueueSize> unacknowledgedSizeRef = new AtomicReference<>(new QueueSize(0, 0L));
-
- // SCHEDULER CANNOT BE NOTIFIED OF EVENTS WITH THE WRITE LOCK HELD! DOING SO WILL RESULT IN A DEADLOCK!
- private final ProcessScheduler scheduler;
-
- public StandardFlowFileQueue(final String identifier, final Connection connection, final FlowFileRepository flowFileRepo, final ProvenanceEventRepository provRepo,
- final ResourceClaimManager resourceClaimManager, final ProcessScheduler scheduler, final FlowFileSwapManager swapManager, final EventReporter eventReporter, final int swapThreshold) {
- activeQueue = new PriorityQueue<>(20, new Prioritizer(new ArrayList<FlowFilePrioritizer>()));
- priorities = new ArrayList<>();
- maximumQueueObjectCount = 0L;
- maximumQueueDataSize = "0 MB";
- maximumQueueByteCount = 0L;
- flowFileExpirationMillis = new AtomicLong(0);
- flowFileExpirationPeriod = new AtomicReference<>("0 mins");
- swapQueue = new ArrayList<>();
- this.eventReporter = eventReporter;
- this.swapManager = swapManager;
- this.flowFileRepository = flowFileRepo;
- this.provRepository = provRepo;
- this.resourceClaimManager = resourceClaimManager;
-
- this.identifier = identifier;
- this.swapThreshold = swapThreshold;
- this.scheduler = scheduler;
- this.connection = connection;
-
- readLock = new TimedLock(this.lock.readLock(), identifier + " Read Lock", 100);
- writeLock = new TimedLock(this.lock.writeLock(), identifier + " Write Lock", 100);
- }
-
- @Override
- public String getIdentifier() {
- return identifier;
- }
-
- @Override
- public List<FlowFilePrioritizer> getPriorities() {
- return Collections.unmodifiableList(priorities);
- }
-
- @Override
- public void setPriorities(final List<FlowFilePrioritizer> newPriorities) {
- writeLock.lock();
- try {
- final PriorityQueue<FlowFileRecord> newQueue = new PriorityQueue<>(Math.max(20, activeQueue.size()), new Prioritizer(newPriorities));
- newQueue.addAll(activeQueue);
- activeQueue = newQueue;
- priorities.clear();
- priorities.addAll(newPriorities);
- } finally {
- writeLock.unlock("setPriorities");
- }
- }
-
- @Override
- public void setBackPressureObjectThreshold(final long maxQueueSize) {
- writeLock.lock();
- try {
- maximumQueueObjectCount = maxQueueSize;
- this.queueFullRef.set(determineIfFull());
- } finally {
- writeLock.unlock("setBackPressureObjectThreshold");
- }
- }
-
- @Override
- public long getBackPressureObjectThreshold() {
- readLock.lock();
- try {
- return maximumQueueObjectCount;
- } finally {
- readLock.unlock("getBackPressureObjectThreshold");
- }
- }
-
- @Override
- public void setBackPressureDataSizeThreshold(final String maxDataSize) {
- writeLock.lock();
- try {
- maximumQueueByteCount = DataUnit.parseDataSize(maxDataSize, DataUnit.B).longValue();
- maximumQueueDataSize = maxDataSize;
- this.queueFullRef.set(determineIfFull());
- } finally {
- writeLock.unlock("setBackPressureDataSizeThreshold");
- }
- }
-
- @Override
- public String getBackPressureDataSizeThreshold() {
- readLock.lock();
- try {
- return maximumQueueDataSize;
- } finally {
- readLock.unlock("getBackPressureDataSizeThreshold");
- }
- }
-
- @Override
- public QueueSize size() {
- readLock.lock();
- try {
- return getQueueSize();
- } finally {
- readLock.unlock("getSize");
- }
- }
-
- /**
- * MUST be called with lock held
- *
- * @return size of queue
- */
- private QueueSize getQueueSize() {
- final QueueSize unacknowledged = unacknowledgedSizeRef.get();
-
- return new QueueSize(activeQueue.size() + swappedRecordCount + unacknowledged.getObjectCount(),
- activeQueueContentSize + swappedContentSize + unacknowledged.getByteCount());
- }
-
- @Override
- public boolean isEmpty() {
- readLock.lock();
- try {
- return activeQueue.isEmpty() && swappedRecordCount == 0 && unacknowledgedSizeRef.get().getObjectCount() == 0;
- } finally {
- readLock.unlock("isEmpty");
- }
- }
-
- @Override
- public boolean isActiveQueueEmpty() {
- final int activeQueueSize = activeQueueSizeRef.get();
- return activeQueueSize == 0;
- }
-
- public QueueSize getActiveQueueSize() {
- readLock.lock();
- try {
- return new QueueSize(activeQueue.size(), activeQueueContentSize);
- } finally {
- readLock.unlock("getActiveQueueSize");
- }
- }
-
- @Override
- public void acknowledge(final FlowFileRecord flowFile) {
- if (queueFullRef.get()) {
- writeLock.lock();
- try {
- updateUnacknowledgedSize(-1, -flowFile.getSize());
- queueFullRef.set(determineIfFull());
- } finally {
- writeLock.unlock("acknowledge(FlowFileRecord)");
- }
- } else {
- updateUnacknowledgedSize(-1, -flowFile.getSize());
- }
-
- if (connection.getSource().getSchedulingStrategy() == SchedulingStrategy.EVENT_DRIVEN) {
- // queue was full but no longer is. Notify that the source may now be available to run,
- // because of back pressure caused by this queue.
- scheduler.registerEvent(connection.getSource());
- }
- }
-
- @Override
- public void acknowledge(final Collection<FlowFileRecord> flowFiles) {
- long totalSize = 0L;
- for (final FlowFileRecord flowFile : flowFiles) {
- totalSize += flowFile.getSize();
- }
-
- if (queueFullRef.get()) {
- writeLock.lock();
- try {
- updateUnacknowledgedSize(-flowFiles.size(), -totalSize);
- queueFullRef.set(determineIfFull());
- } finally {
- writeLock.unlock("acknowledge(FlowFileRecord)");
- }
- } else {
- updateUnacknowledgedSize(-flowFiles.size(), -totalSize);
- }
-
- if (connection.getSource().getSchedulingStrategy() == SchedulingStrategy.EVENT_DRIVEN) {
- // it's possible that queue was full but no longer is. Notify that the source may now be available to run,
- // because of back pressure caused by this queue.
- scheduler.registerEvent(connection.getSource());
- }
- }
-
- @Override
- public boolean isFull() {
- return queueFullRef.get();
- }
-
- /**
- * MUST be called with either the read or write lock held
- *
- * @return true if full
- */
- private boolean determineIfFull() {
- final long maxSize = maximumQueueObjectCount;
- final long maxBytes = maximumQueueByteCount;
- if (maxSize <= 0 && maxBytes <= 0) {
- return false;
- }
-
- final QueueSize queueSize = getQueueSize();
- if (maxSize > 0 && queueSize.getObjectCount() >= maxSize) {
- return true;
- }
-
- if (maxBytes > 0 && queueSize.getByteCount() >= maxBytes) {
- return true;
- }
-
- return false;
- }
-
- @Override
- public void put(final FlowFileRecord file) {
- writeLock.lock();
- try {
- if (swapMode || activeQueue.size() >= swapThreshold) {
- swapQueue.add(file);
- swappedContentSize += file.getSize();
- swappedRecordCount++;
- swapMode = true;
- writeSwapFilesIfNecessary();
- } else {
- activeQueueContentSize += file.getSize();
- activeQueue.add(file);
- }
-
- queueFullRef.set(determineIfFull());
- } finally {
- activeQueueSizeRef.set(activeQueue.size());
- writeLock.unlock("put(FlowFileRecord)");
- }
-
- if (connection.getDestination().getSchedulingStrategy() == SchedulingStrategy.EVENT_DRIVEN) {
- scheduler.registerEvent(connection.getDestination());
- }
- }
-
- @Override
- public void putAll(final Collection<FlowFileRecord> files) {
- final int numFiles = files.size();
- long bytes = 0L;
- for (final FlowFile flowFile : files) {
- bytes += flowFile.getSize();
- }
-
- writeLock.lock();
- try {
- if (swapMode || activeQueue.size() >= swapThreshold - numFiles) {
- swapQueue.addAll(files);
- swappedContentSize += bytes;
- swappedRecordCount += numFiles;
- swapMode = true;
- writeSwapFilesIfNecessary();
- } else {
- activeQueueContentSize += bytes;
- activeQueue.addAll(files);
- }
-
- queueFullRef.set(determineIfFull());
- } finally {
- activeQueueSizeRef.set(activeQueue.size());
- writeLock.unlock("putAll");
- }
-
- if (connection.getDestination().getSchedulingStrategy() == SchedulingStrategy.EVENT_DRIVEN) {
- scheduler.registerEvent(connection.getDestination());
- }
- }
-
-
- private boolean isLaterThan(final Long maxAge) {
- if (maxAge == null) {
- return false;
- }
- return maxAge < System.currentTimeMillis();
- }
-
- private Long getExpirationDate(final FlowFile flowFile, final long expirationMillis) {
- if (flowFile == null) {
- return null;
- }
- if (expirationMillis <= 0) {
- return null;
- } else {
- final long entryDate = flowFile.getEntryDate();
- final long expirationDate = entryDate + expirationMillis;
- return expirationDate;
- }
- }
-
- @Override
- public FlowFileRecord poll(final Set<FlowFileRecord> expiredRecords) {
- FlowFileRecord flowFile = null;
-
- // First check if we have any records Pre-Fetched.
- final long expirationMillis = flowFileExpirationMillis.get();
- writeLock.lock();
- try {
- flowFile = doPoll(expiredRecords, expirationMillis);
- return flowFile;
- } finally {
- activeQueueSizeRef.set(activeQueue.size());
- writeLock.unlock("poll(Set)");
-
- if (flowFile != null) {
- updateUnacknowledgedSize(1, flowFile.getSize());
- }
- }
- }
-
- private FlowFileRecord doPoll(final Set<FlowFileRecord> expiredRecords, final long expirationMillis) {
- FlowFileRecord flowFile;
- boolean isExpired;
-
- migrateSwapToActive();
- final boolean queueFullAtStart = queueFullRef.get();
-
- do {
- flowFile = this.activeQueue.poll();
-
- isExpired = isLaterThan(getExpirationDate(flowFile, expirationMillis));
- if (isExpired) {
- expiredRecords.add(flowFile);
- if (expiredRecords.size() >= MAX_EXPIRED_RECORDS_PER_ITERATION) {
- activeQueueContentSize -= flowFile.getSize();
- break;
- }
- } else if (flowFile != null && flowFile.isPenalized()) {
- this.activeQueue.add(flowFile);
- flowFile = null;
- break;
- }
-
- if (flowFile != null) {
- activeQueueContentSize -= flowFile.getSize();
- }
- } while (isExpired);
-
- // if at least 1 FlowFile was expired & the queue was full before we started, then
- // we need to determine whether or not the queue is full again. If no FlowFile was expired,
- // then the queue will still be full until the appropriate #acknowledge method is called.
- if (queueFullAtStart && !expiredRecords.isEmpty()) {
- queueFullRef.set(determineIfFull());
- }
-
- return isExpired ? null : flowFile;
- }
-
- @Override
- public List<FlowFileRecord> poll(int maxResults, final Set<FlowFileRecord> expiredRecords) {
- final List<FlowFileRecord> records = new ArrayList<>(Math.min(1024, maxResults));
-
- // First check if we have any records Pre-Fetched.
- writeLock.lock();
- try {
- doPoll(records, maxResults, expiredRecords);
- } finally {
- activeQueueSizeRef.set(activeQueue.size());
- writeLock.unlock("poll(int, Set)");
- }
- return records;
- }
-
- private void doPoll(final List<FlowFileRecord> records, int maxResults, final Set<FlowFileRecord> expiredRecords) {
- migrateSwapToActive();
-
- final boolean queueFullAtStart = queueFullRef.get();
-
- final long bytesDrained = drainQueue(activeQueue, records, maxResults, expiredRecords);
-
- long expiredBytes = 0L;
- for (final FlowFileRecord record : expiredRecords) {
- expiredBytes += record.getSize();
- }
-
- activeQueueContentSize -= bytesDrained;
- updateUnacknowledgedSize(records.size(), bytesDrained - expiredBytes);
-
- // if at least 1 FlowFile was expired & the queue was full before we started, then
- // we need to determine whether or not the queue is full again. If no FlowFile was expired,
- // then the queue will still be full until the appropriate #acknowledge method is called.
- if (queueFullAtStart && !expiredRecords.isEmpty()) {
- queueFullRef.set(determineIfFull());
- }
- }
-
- /**
- * If there are FlowFiles waiting on the swap queue, move them to the active
- * queue until we meet our threshold. This prevents us from having to swap
- * them to disk & then back out.
- *
- * This method MUST be called with the writeLock held.
- */
- private void migrateSwapToActive() {
- // Migrate as many FlowFiles as we can from the Swap Queue to the Active Queue, so that we don't
- // have to swap them out & then swap them back in.
- // If we don't do this, we could get into a situation where we have potentially thousands of FlowFiles
- // sitting on the Swap Queue but not getting processed because there aren't enough to be swapped out.
- // In particular, this can happen if the queue is typically filled with surges.
- // For example, if the queue has 25,000 FlowFiles come in, it may process 20,000 of them and leave
- // 5,000 sitting on the Swap Queue. If it then takes an hour for an additional 5,000 FlowFiles to come in,
- // those FlowFiles sitting on the Swap Queue will sit there for an hour, waiting to be swapped out and
- // swapped back in again.
- // Calling this method when records are polled prevents this condition by migrating FlowFiles from the
- // Swap Queue to the Active Queue. However, we don't do this if there are FlowFiles already swapped out
- // to disk, because we want them to be swapped back in in the same order that they were swapped out.
-
- if (activeQueue.size() > swapThreshold - SWAP_RECORD_POLL_SIZE) {
- return;
- }
-
- // If there are swap files waiting to be swapped in, swap those in first. We do this in order to ensure that those that
- // were swapped out first are then swapped back in first. If we instead just immediately migrated the FlowFiles from the
- // swap queue to the active queue, and we never run out of FlowFiles in the active queue (because destination cannot
- // keep up with queue), we will end up always processing the new FlowFiles first instead of the FlowFiles that arrived
- // first.
- if (!swapLocations.isEmpty()) {
- final String swapLocation = swapLocations.remove(0);
- try {
- final List<FlowFileRecord> swappedIn = swapManager.swapIn(swapLocation, this);
- swappedRecordCount -= swappedIn.size();
- long swapSize = 0L;
- for (final FlowFileRecord flowFile : swappedIn) {
- swapSize += flowFile.getSize();
- }
- swappedContentSize -= swapSize;
- activeQueueContentSize += swapSize;
- activeQueueSizeRef.set(activeQueue.size());
- activeQueue.addAll(swappedIn);
- return;
- } catch (final FileNotFoundException fnfe) {
- logger.error("Failed to swap in FlowFiles from Swap File {} because the Swap File can no longer be found", swapLocation);
- if (eventReporter != null) {
- eventReporter.reportEvent(Severity.ERROR, "Swap File", "Failed to swap in FlowFiles from Swap File " + swapLocation + " because the Swap File can no longer be found");
- }
- return;
- } catch (final IOException ioe) {
- logger.error("Failed to swap in FlowFiles from Swap File {}; Swap File appears to be corrupt!", swapLocation);
- logger.error("", ioe);
- if (eventReporter != null) {
- eventReporter.reportEvent(Severity.ERROR, "Swap File", "Failed to swap in FlowFiles from Swap File " +
- swapLocation + "; Swap File appears to be corrupt! Some FlowFiles in the queue may not be accessible. See logs for more information.");
- }
- return;
- }
- }
-
- // this is the most common condition (nothing is swapped out), so do the check first and avoid the expense
- // of other checks for 99.999% of the cases.
- if (swappedRecordCount == 0 && swapQueue.isEmpty()) {
- return;
- }
-
- if (swappedRecordCount > swapQueue.size()) {
- // we already have FlowFiles swapped out, so we won't migrate the queue; we will wait for
- // an external process to swap FlowFiles back in.
- return;
- }
-
- final Iterator<FlowFileRecord> swapItr = swapQueue.iterator();
- while (activeQueue.size() < swapThreshold && swapItr.hasNext()) {
- final FlowFileRecord toMigrate = swapItr.next();
- activeQueue.add(toMigrate);
- activeQueueContentSize += toMigrate.getSize();
- swappedContentSize -= toMigrate.getSize();
- swappedRecordCount--;
-
- swapItr.remove();
- }
-
- if (swappedRecordCount == 0) {
- swapMode = false;
- }
- }
-
- /**
- * This method MUST be called with the write lock held
- */
- private void writeSwapFilesIfNecessary() {
- if (swapQueue.size() < SWAP_RECORD_POLL_SIZE) {
- return;
- }
-
- final int numSwapFiles = swapQueue.size() / SWAP_RECORD_POLL_SIZE;
-
- // Create a new Priority queue with the prioritizers that are set, but reverse the
- // prioritizers because we want to pull the lowest-priority FlowFiles to swap out
- final PriorityQueue<FlowFileRecord> tempQueue = new PriorityQueue<>(activeQueue.size() + swapQueue.size(), Collections.reverseOrder(new Prioritizer(priorities)));
- tempQueue.addAll(activeQueue);
- tempQueue.addAll(swapQueue);
-
- final List<String> swapLocations = new ArrayList<>(numSwapFiles);
- for (int i = 0; i < numSwapFiles; i++) {
- // Create a new swap file for the next SWAP_RECORD_POLL_SIZE records
- final List<FlowFileRecord> toSwap = new ArrayList<>(SWAP_RECORD_POLL_SIZE);
- for (int j = 0; j < SWAP_RECORD_POLL_SIZE; j++) {
- toSwap.add(tempQueue.poll());
- }
-
- try {
- Collections.reverse(toSwap); // currently ordered in reverse priority order based on the ordering of the temp queue.
- final String swapLocation = swapManager.swapOut(toSwap, this);
- swapLocations.add(swapLocation);
- } catch (final IOException ioe) {
- tempQueue.addAll(toSwap); // if we failed, we must add the FlowFiles back to the queue.
- logger.error("FlowFile Queue with identifier {} has {} FlowFiles queued up. Attempted to spill FlowFile information over to disk in order to avoid exhausting "
- + "the Java heap space but failed to write information to disk due to {}", getIdentifier(), getQueueSize().getObjectCount(), ioe.toString());
- logger.error("", ioe);
- if (eventReporter != null) {
- eventReporter.reportEvent(Severity.ERROR, "Failed to Overflow to Disk", "Flowfile Queue with identifier " + getIdentifier() + " has " + getQueueSize().getObjectCount() +
- " queued up. Attempted to spill FlowFile information over to disk in order to avoid exhausting the Java heap space but failed to write information to disk. "
- + "See logs for more information.");
- }
-
- break;
- }
- }
-
- // Pull any records off of the temp queue that won't fit back on the active queue, and add those to the
- // swap queue. Then add the records back to the active queue.
- swapQueue.clear();
- while (tempQueue.size() > swapThreshold) {
- final FlowFileRecord record = tempQueue.poll();
- swapQueue.add(record);
- }
-
- Collections.reverse(swapQueue); // currently ordered in reverse priority order based on the ordering of the temp queue
-
- // replace the contents of the active queue, since we've merged it with the swap queue.
- activeQueue.clear();
- FlowFileRecord toRequeue;
- while ((toRequeue = tempQueue.poll()) != null) {
- activeQueue.offer(toRequeue);
- }
- this.swapLocations.addAll(swapLocations);
- }
-
-
- @Override
- public long drainQueue(final Queue<FlowFileRecord> sourceQueue, final List<FlowFileRecord> destination, int maxResults, final Set<FlowFileRecord> expiredRecords) {
- long drainedSize = 0L;
- FlowFileRecord pulled = null;
-
- final long expirationMillis = this.flowFileExpirationMillis.get();
- while (destination.size() < maxResults && (pulled = sourceQueue.poll()) != null) {
- if (isLaterThan(getExpirationDate(pulled, expirationMillis))) {
- expiredRecords.add(pulled);
- if (expiredRecords.size() >= MAX_EXPIRED_RECORDS_PER_ITERATION) {
- break;
- }
- } else {
- if (pulled.isPenalized()) {
- sourceQueue.add(pulled);
- break;
- }
- destination.add(pulled);
- }
- drainedSize += pulled.getSize();
- }
- return drainedSize;
- }
-
- @Override
- public List<FlowFileRecord> poll(final FlowFileFilter filter, final Set<FlowFileRecord> expiredRecords) {
- writeLock.lock();
- try {
- migrateSwapToActive();
-
- final long expirationMillis = this.flowFileExpirationMillis.get();
- final boolean queueFullAtStart = queueFullRef.get();
-
- final List<FlowFileRecord> selectedFlowFiles = new ArrayList<>();
- final List<FlowFileRecord> unselected = new ArrayList<>();
-
- while (true) {
- FlowFileRecord flowFile = this.activeQueue.poll();
- if (flowFile == null) {
- break;
- }
-
- final boolean isExpired = isLaterThan(getExpirationDate(flowFile, expirationMillis));
- if (isExpired) {
- expiredRecords.add(flowFile);
- activeQueueContentSize -= flowFile.getSize();
-
- if (expiredRecords.size() >= MAX_EXPIRED_RECORDS_PER_ITERATION) {
- break;
- } else {
- continue;
- }
- } else if (flowFile.isPenalized()) {
- this.activeQueue.add(flowFile);
- flowFile = null;
- break; // just stop searching because the rest are all penalized.
- }
-
- final FlowFileFilterResult result = filter.filter(flowFile);
- if (result.isAccept()) {
- activeQueueContentSize -= flowFile.getSize();
-
- updateUnacknowledgedSize(1, flowFile.getSize());
- selectedFlowFiles.add(flowFile);
- } else {
- unselected.add(flowFile);
- }
-
- if (!result.isContinue()) {
- break;
- }
- }
-
- this.activeQueue.addAll(unselected);
-
- // if at least 1 FlowFile was expired & the queue was full before we started, then
- // we need to determine whether or not the queue is full again. If no FlowFile was expired,
- // then the queue will still be full until the appropriate #acknowledge method is called.
- if (queueFullAtStart && !expiredRecords.isEmpty()) {
- queueFullRef.set(determineIfFull());
- }
-
- return selectedFlowFiles;
- } finally {
- activeQueueSizeRef.set(activeQueue.size());
- writeLock.unlock("poll(Filter, Set)");
- }
- }
-
-
-
- private static final class Prioritizer implements Comparator<FlowFileRecord>, Serializable {
-
- private static final long serialVersionUID = 1L;
- private final transient List<FlowFilePrioritizer> prioritizers = new ArrayList<>();
-
- private Prioritizer(final List<FlowFilePrioritizer> priorities) {
- if (null != priorities) {
- prioritizers.addAll(priorities);
- }
- }
-
- @Override
- public int compare(final FlowFileRecord f1, final FlowFileRecord f2) {
- int returnVal = 0;
- final boolean f1Penalized = f1.isPenalized();
- final boolean f2Penalized = f2.isPenalized();
-
- if (f1Penalized && !f2Penalized) {
- return 1;
- } else if (!f1Penalized && f2Penalized) {
- return -1;
- }
-
- if (f1Penalized && f2Penalized) {
- if (f1.getPenaltyExpirationMillis() < f2.getPenaltyExpirationMillis()) {
- return -1;
- } else if (f1.getPenaltyExpirationMillis() > f2.getPenaltyExpirationMillis()) {
- return 1;
- }
- }
-
- if (!prioritizers.isEmpty()) {
- for (final FlowFilePrioritizer prioritizer : prioritizers) {
- returnVal = prioritizer.compare(f1, f2);
- if (returnVal != 0) {
- return returnVal;
- }
- }
- }
-
- final ContentClaim claim1 = f1.getContentClaim();
- final ContentClaim claim2 = f2.getContentClaim();
-
- // put the one without a claim first
- if (claim1 == null && claim2 != null) {
- return -1;
- } else if (claim1 != null && claim2 == null) {
- return 1;
- } else if (claim1 != null && claim2 != null) {
- final int claimComparison = claim1.compareTo(claim2);
- if (claimComparison != 0) {
- return claimComparison;
- }
-
- final int claimOffsetComparison = Long.compare(f1.getContentClaimOffset(), f2.getContentClaimOffset());
- if (claimOffsetComparison != 0) {
- return claimOffsetComparison;
- }
- }
-
- return Long.compare(f1.getId(), f2.getId());
- }
- }
-
- @Override
- public String getFlowFileExpiration() {
- return flowFileExpirationPeriod.get();
- }
-
- @Override
- public int getFlowFileExpiration(final TimeUnit timeUnit) {
- return (int) timeUnit.convert(flowFileExpirationMillis.get(), TimeUnit.MILLISECONDS);
- }
-
- @Override
- public void setFlowFileExpiration(final String flowExpirationPeriod) {
- final long millis = FormatUtils.getTimeDuration(flowExpirationPeriod, TimeUnit.MILLISECONDS);
- if (millis < 0) {
- throw new IllegalArgumentException("FlowFile Expiration Period must be positive");
- }
- this.flowFileExpirationPeriod.set(flowExpirationPeriod);
- this.flowFileExpirationMillis.set(millis);
- }
-
-
- @Override
- public void purgeSwapFiles() {
- swapManager.purge();
- }
-
- @Override
- public Long recoverSwappedFlowFiles() {
- int swapFlowFileCount = 0;
- long swapByteCount = 0L;
- Long maxId = null;
-
- writeLock.lock();
- try {
- final List<String> swapLocations;
- try {
- swapLocations = swapManager.recoverSwapLocations(this);
- } catch (final IOException ioe) {
- logger.error("Failed to determine whether or not any Swap Files exist for FlowFile Queue {}", getIdentifier());
- logger.error("", ioe);
- if (eventReporter != null) {
- eventReporter.reportEvent(Severity.ERROR, "FlowFile Swapping", "Failed to determine whether or not any Swap Files exist for FlowFile Queue " +
- getIdentifier() + "; see logs for more detials");
- }
- return null;
- }
-
- for (final String swapLocation : swapLocations) {
- try {
- final QueueSize queueSize = swapManager.getSwapSize(swapLocation);
- final Long maxSwapRecordId = swapManager.getMaxRecordId(swapLocation);
- if (maxSwapRecordId != null) {
- if (maxId == null || maxSwapRecordId > maxId) {
- maxId = maxSwapRecordId;
- }
- }
-
- swapFlowFileCount += queueSize.getObjectCount();
- swapByteCount += queueSize.getByteCount();
- } catch (final IOException ioe) {
- logger.error("Failed to recover FlowFiles from Swap File {}; the file appears to be corrupt", swapLocation, ioe.toString());
- logger.error("", ioe);
- if (eventReporter != null) {
- eventReporter.reportEvent(Severity.ERROR, "FlowFile Swapping", "Failed to recover FlowFiles from Swap File " + swapLocation +
- "; the file appears to be corrupt. See logs for more details");
- }
- }
- }
-
- this.swappedRecordCount = swapFlowFileCount;
- this.swappedContentSize = swapByteCount;
- this.swapLocations.addAll(swapLocations);
- } finally {
- writeLock.unlock("Recover Swap Files");
- }
-
- return maxId;
- }
-
-
- @Override
- public String toString() {
- return "FlowFileQueue[id=" + identifier + "]";
- }
-
- private final ConcurrentMap<String, DropFlowFileRequest> dropRequestMap = new ConcurrentHashMap<>();
-
- @Override
- public DropFlowFileStatus dropFlowFiles(final String requestIdentifier, final String requestor) {
- // purge any old requests from the map just to keep it clean. But if there are very requests, which is usually the case, then don't bother
- if (dropRequestMap.size() > 10) {
- final List<String> toDrop = new ArrayList<>();
- for (final Map.Entry<String, DropFlowFileRequest> entry : dropRequestMap.entrySet()) {
- final DropFlowFileRequest request = entry.getValue();
- final boolean completed = request.getState() == DropFlowFileState.COMPLETE || request.getState() == DropFlowFileState.FAILURE;
-
- if (completed && System.currentTimeMillis() - request.getLastUpdated() > TimeUnit.MINUTES.toMillis(5L)) {
- toDrop.add(entry.getKey());
- }
- }
-
- for (final String requestId : toDrop) {
- dropRequestMap.remove(requestId);
- }
- }
-
- final DropFlowFileRequest dropRequest = new DropFlowFileRequest(requestIdentifier);
- final Thread t = new Thread(new Runnable() {
- @Override
- public void run() {
- writeLock.lock();
- try {
- dropRequest.setState(DropFlowFileState.DROPPING_ACTIVE_FLOWFILES);
- dropRequest.setOriginalSize(getQueueSize());
-
- try {
- final List<FlowFileRecord> activeQueueRecords = new ArrayList<>(activeQueue);
-
- QueueSize droppedSize;
- try {
- droppedSize = drop(activeQueueRecords, requestor);
- } catch (final IOException ioe) {
- logger.error("Failed to drop the FlowFiles from queue {} due to {}", StandardFlowFileQueue.this.getIdentifier(), ioe.toString());
- logger.error("", ioe);
-
- dropRequest.setState(DropFlowFileState.FAILURE, "Failed to drop FlowFiles due to " + ioe.toString());
- return;
- }
-
- activeQueue.clear();
- activeQueueContentSize = 0;
- activeQueueSizeRef.set(0);
- dropRequest.setCurrentSize(getQueueSize());
- dropRequest.setDroppedSize(dropRequest.getDroppedSize().add(droppedSize));
-
- try {
- droppedSize = drop(swapQueue, requestor);
- } catch (final IOException ioe) {
- logger.error("Failed to drop the FlowFiles from queue {} due to {}", StandardFlowFileQueue.this.getIdentifier(), ioe.toString());
- logger.error("", ioe);
-
- dropRequest.setState(DropFlowFileState.FAILURE, "Failed to drop FlowFiles due to " + ioe.toString());
- return;
- }
-
- swapQueue.clear();
- dropRequest.setCurrentSize(getQueueSize());
- dropRequest.setDroppedSize(dropRequest.getDroppedSize().add(droppedSize));
- swapMode = false;
- swappedContentSize -= droppedSize.getByteCount();
- swappedRecordCount -= droppedSize.getObjectCount();
-
- final Iterator<String> swapLocationItr = swapLocations.iterator();
- while (swapLocationItr.hasNext()) {
- final String swapLocation = swapLocationItr.next();
-
- List<FlowFileRecord> swappedIn = null;
- try {
- swappedIn = swapManager.swapIn(swapLocation, StandardFlowFileQueue.this);
- droppedSize = drop(swappedIn, requestor);
- } catch (final IOException ioe) {
- logger.error("Failed to swap in FlowFiles from Swap File {} in order to drop the FlowFiles for Connection {} due to {}",
- swapLocation, StandardFlowFileQueue.this.getIdentifier(), ioe.toString());
- logger.error("", ioe);
-
- dropRequest.setState(DropFlowFileState.FAILURE, "Failed to swap in FlowFiles from Swap File " + swapLocation + " due to " + ioe.toString());
- if (swappedIn != null) {
- activeQueue.addAll(swappedIn); // ensure that we don't lose the FlowFiles from our queue.
- }
-
- return;
- }
-
- dropRequest.setDroppedSize(dropRequest.getDroppedSize().add(droppedSize));
- swappedContentSize -= droppedSize.getByteCount();
- swappedRecordCount -= droppedSize.getObjectCount();
- dropRequest.setCurrentSize(getQueueSize());
- swapLocationItr.remove();
- }
-
- dropRequest.setState(DropFlowFileState.COMPLETE);
- } catch (final Exception e) {
- dropRequest.setState(DropFlowFileState.FAILURE, "Failed to drop FlowFiles due to " + e.toString());
- }
- } finally {
- writeLock.unlock("Drop FlowFiles");
- }
- }
- }, "Drop FlowFiles for Connection " + getIdentifier());
- t.setDaemon(true);
- t.start();
-
- dropRequest.setExecutionThread(t);
- dropRequestMap.put(requestIdentifier, dropRequest);
-
- return dropRequest;
- }
-
- private QueueSize drop(final List<FlowFileRecord> flowFiles, final String requestor) throws IOException {
- // Create a Provenance Event and a FlowFile Repository record for each FlowFile
- final List<ProvenanceEventRecord> provenanceEvents = new ArrayList<>(flowFiles.size());
- final List<RepositoryRecord> flowFileRepoRecords = new ArrayList<>(flowFiles.size());
- for (final FlowFileRecord flowFile : flowFiles) {
- provenanceEvents.add(createDropEvent(flowFile, requestor));
- flowFileRepoRecords.add(createDeleteRepositoryRecord(flowFile));
- }
-
- long dropContentSize = 0L;
- for (final FlowFileRecord flowFile : flowFiles) {
- final ContentClaim contentClaim = flowFile.getContentClaim();
- if (contentClaim == null) {
- continue;
- }
-
- final ResourceClaim resourceClaim = contentClaim.getResourceClaim();
- if (resourceClaim == null) {
- continue;
- }
-
- resourceClaimManager.decrementClaimantCount(resourceClaim);
- dropContentSize += flowFile.getSize();
- }
-
- provRepository.registerEvents(provenanceEvents);
- flowFileRepository.updateRepository(flowFileRepoRecords);
- return new QueueSize(flowFiles.size(), dropContentSize);
- }
-
- private ProvenanceEventRecord createDropEvent(final FlowFileRecord flowFile, final String requestor) {
- final ProvenanceEventBuilder builder = provRepository.eventBuilder();
- builder.fromFlowFile(flowFile);
- builder.setEventType(ProvenanceEventType.DROP);
- builder.setLineageStartDate(flowFile.getLineageStartDate());
- builder.setComponentId(getIdentifier());
- builder.setComponentType("Connection");
- builder.setDetails("FlowFile manually dropped; request made by " + requestor);
- return builder.build();
- }
-
- private RepositoryRecord createDeleteRepositoryRecord(final FlowFileRecord flowFile) {
- return new RepositoryRecord() {
- @Override
- public FlowFileQueue getDestination() {
- return null;
- }
-
- @Override
- public FlowFileQueue getOriginalQueue() {
- return StandardFlowFileQueue.this;
- }
-
- @Override
- public RepositoryRecordType getType() {
- return RepositoryRecordType.DELETE;
- }
-
- @Override
- public ContentClaim getCurrentClaim() {
- return flowFile.getContentClaim();
- }
-
- @Override
- public ContentClaim getOriginalClaim() {
- return flowFile.getContentClaim();
- }
-
- @Override
- public long getCurrentClaimOffset() {
- return flowFile.getContentClaimOffset();
- }
-
- @Override
- public FlowFileRecord getCurrent() {
- return flowFile;
- }
-
- @Override
- public boolean isAttributesChanged() {
- return false;
- }
-
- @Override
- public boolean isMarkedForAbort() {
- return false;
- }
-
- @Override
- public String getSwapLocation() {
- return null;
- }
- };
- }
-
-
- @Override
- public boolean cancelDropFlowFileRequest(final String requestIdentifier) {
- final DropFlowFileRequest request = dropRequestMap.remove(requestIdentifier);
- if (request == null) {
- return false;
- }
-
- final boolean successful = request.cancel();
- return successful;
- }
-
- @Override
- public DropFlowFileStatus getDropFlowFileStatus(final String requestIdentifier) {
- return dropRequestMap.get(requestIdentifier);
- }
-
- /**
- * Lock the queue so that other threads are unable to interact with the
- * queue
- */
- public void lock() {
- writeLock.lock();
- }
-
- /**
- * Unlock the queue
- */
- public void unlock() {
- writeLock.unlock("external unlock");
- }
-
- @Override
- public QueueSize getUnacknowledgedQueueSize() {
- return unacknowledgedSizeRef.get();
- }
-
- private void updateUnacknowledgedSize(final int addToCount, final long addToSize) {
- boolean updated = false;
-
- do {
- final QueueSize queueSize = unacknowledgedSizeRef.get();
- final QueueSize newSize = new QueueSize(queueSize.getObjectCount() + addToCount, queueSize.getByteCount() + addToSize);
- updated = unacknowledgedSizeRef.compareAndSet(queueSize, newSize);
- } while (!updated);
- }
-}
http://git-wip-us.apache.org/repos/asf/nifi/blob/77f7d752/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/test/java/org/apache/nifi/controller/TestStandardFlowFileQueue.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/test/java/org/apache/nifi/controller/TestStandardFlowFileQueue.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/test/java/org/apache/nifi/controller/TestStandardFlowFileQueue.java
deleted file mode 100644
index 66f32d8..0000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/test/java/org/apache/nifi/controller/TestStandardFlowFileQueue.java
+++ /dev/null
@@ -1,330 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nifi.controller;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.atomic.AtomicLong;
-
-import org.apache.nifi.connectable.Connectable;
-import org.apache.nifi.connectable.Connection;
-import org.apache.nifi.controller.queue.FlowFileQueue;
-import org.apache.nifi.controller.queue.QueueSize;
-import org.apache.nifi.controller.repository.FlowFileRecord;
-import org.apache.nifi.controller.repository.FlowFileRepository;
-import org.apache.nifi.controller.repository.FlowFileSwapManager;
-import org.apache.nifi.controller.repository.SwapManagerInitializationContext;
-import org.apache.nifi.controller.repository.claim.ContentClaim;
-import org.apache.nifi.controller.repository.claim.ResourceClaimManager;
-import org.apache.nifi.flowfile.FlowFile;
-import org.apache.nifi.flowfile.FlowFilePrioritizer;
-import org.apache.nifi.provenance.ProvenanceEventRepository;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mockito;
-
-public class TestStandardFlowFileQueue {
- private TestSwapManager swapManager = null;
- private StandardFlowFileQueue queue = null;
-
- @Before
- public void setup() {
- final Connection connection = Mockito.mock(Connection.class);
- Mockito.when(connection.getSource()).thenReturn(Mockito.mock(Connectable.class));
- Mockito.when(connection.getDestination()).thenReturn(Mockito.mock(Connectable.class));
-
- final ProcessScheduler scheduler = Mockito.mock(ProcessScheduler.class);
- swapManager = new TestSwapManager();
-
- final FlowFileRepository flowFileRepo = Mockito.mock(FlowFileRepository.class);
- final ProvenanceEventRepository provRepo = Mockito.mock(ProvenanceEventRepository.class);
- final ResourceClaimManager claimManager = Mockito.mock(ResourceClaimManager.class);
-
- queue = new StandardFlowFileQueue("id", connection, flowFileRepo, provRepo, claimManager, scheduler, swapManager, null, 10000);
- TestFlowFile.idGenerator.set(0L);
- }
-
-
- @Test
- public void testSwapOutOccurs() {
- for (int i = 0; i < 10000; i++) {
- queue.put(new TestFlowFile());
- assertEquals(0, swapManager.swapOutCalledCount);
- assertEquals(i + 1, queue.size().getObjectCount());
- assertEquals(i + 1, queue.size().getByteCount());
- }
-
- for (int i = 0; i < 9999; i++) {
- queue.put(new TestFlowFile());
- assertEquals(0, swapManager.swapOutCalledCount);
- assertEquals(i + 10001, queue.size().getObjectCount());
- assertEquals(i + 10001, queue.size().getByteCount());
- }
-
- queue.put(new TestFlowFile(1000));
- assertEquals(1, swapManager.swapOutCalledCount);
- assertEquals(20000, queue.size().getObjectCount());
- assertEquals(20999, queue.size().getByteCount());
-
- assertEquals(10000, queue.getActiveQueueSize().getObjectCount());
- }
-
- @Test
- public void testLowestPrioritySwappedOutFirst() {
- final List<FlowFilePrioritizer> prioritizers = new ArrayList<>();
- prioritizers.add(new FlowFileSizePrioritizer());
- queue.setPriorities(prioritizers);
-
- long maxSize = 20000;
- for (int i = 1; i <= 20000; i++) {
- queue.put(new TestFlowFile(maxSize - i));
- }
-
- assertEquals(1, swapManager.swapOutCalledCount);
- assertEquals(20000, queue.size().getObjectCount());
-
- assertEquals(10000, queue.getActiveQueueSize().getObjectCount());
- final List<FlowFileRecord> flowFiles = queue.poll(Integer.MAX_VALUE, new HashSet<FlowFileRecord>());
- assertEquals(10000, flowFiles.size());
- for (int i = 0; i < 10000; i++) {
- assertEquals(i, flowFiles.get(i).getSize());
- }
- }
-
- @Test
- public void testSwapIn() {
- for (int i = 1; i <= 20000; i++) {
- queue.put(new TestFlowFile());
- }
-
- assertEquals(1, swapManager.swappedOut.size());
- queue.put(new TestFlowFile());
- assertEquals(1, swapManager.swappedOut.size());
-
- final Set<FlowFileRecord> exp = new HashSet<>();
- for (int i = 0; i < 9999; i++) {
- assertNotNull(queue.poll(exp));
- }
-
- assertEquals(0, swapManager.swapInCalledCount);
- assertEquals(1, queue.getActiveQueueSize().getObjectCount());
- assertNotNull(queue.poll(exp));
-
- assertEquals(0, swapManager.swapInCalledCount);
- assertEquals(0, queue.getActiveQueueSize().getObjectCount());
-
- assertEquals(1, swapManager.swapOutCalledCount);
-
- assertNotNull(queue.poll(exp)); // this should trigger a swap-in of 10,000 records, and then pull 1 off the top.
- assertEquals(1, swapManager.swapInCalledCount);
- assertEquals(9999, queue.getActiveQueueSize().getObjectCount());
-
- assertTrue(swapManager.swappedOut.isEmpty());
-
- queue.poll(exp);
-
- }
-
-
- private class TestSwapManager implements FlowFileSwapManager {
- private final Map<String, List<FlowFileRecord>> swappedOut = new HashMap<>();
- int swapOutCalledCount = 0;
- int swapInCalledCount = 0;
-
-
- @Override
- public void initialize(final SwapManagerInitializationContext initializationContext) {
-
- }
-
- @Override
- public String swapOut(List<FlowFileRecord> flowFiles, FlowFileQueue flowFileQueue) throws IOException {
- swapOutCalledCount++;
- final String location = UUID.randomUUID().toString();
- swappedOut.put(location, new ArrayList<FlowFileRecord>(flowFiles));
- return location;
- }
-
- @Override
- public List<FlowFileRecord> peek(String swapLocation, final FlowFileQueue flowFileQueue) throws IOException {
- return new ArrayList<FlowFileRecord>(swappedOut.get(swapLocation));
- }
-
- @Override
- public List<FlowFileRecord> swapIn(String swapLocation, FlowFileQueue flowFileQueue) throws IOException {
- swapInCalledCount++;
- return swappedOut.remove(swapLocation);
- }
-
- @Override
- public List<String> recoverSwapLocations(FlowFileQueue flowFileQueue) throws IOException {
- return new ArrayList<String>(swappedOut.keySet());
- }
-
- @Override
- public void dropSwappedFlowFiles(String swapLocation, final FlowFileQueue flowFileQueue, String user) {
-
- }
-
- @Override
- public QueueSize getSwapSize(String swapLocation) throws IOException {
- final List<FlowFileRecord> flowFiles = swappedOut.get(swapLocation);
- if (flowFiles == null) {
- return new QueueSize(0, 0L);
- }
-
- int count = 0;
- long size = 0L;
- for (final FlowFileRecord flowFile : flowFiles) {
- count++;
- size += flowFile.getSize();
- }
-
- return new QueueSize(count, size);
- }
-
- @Override
- public Long getMaxRecordId(String swapLocation) throws IOException {
- final List<FlowFileRecord> flowFiles = swappedOut.get(swapLocation);
- if (flowFiles == null) {
- return null;
- }
-
- Long max = null;
- for (final FlowFileRecord flowFile : flowFiles) {
- if (max == null || flowFile.getId() > max) {
- max = flowFile.getId();
- }
- }
-
- return max;
- }
-
- @Override
- public void purge() {
- swappedOut.clear();
- }
- }
-
-
- private static class TestFlowFile implements FlowFileRecord {
- private static final AtomicLong idGenerator = new AtomicLong(0L);
-
- private final long id = idGenerator.getAndIncrement();
- private final long entryDate = System.currentTimeMillis();
- private final Map<String, String> attributes;
- private final long size;
-
- public TestFlowFile() {
- this(1L);
- }
-
- public TestFlowFile(final long size) {
- this(new HashMap<String, String>(), size);
- }
-
- public TestFlowFile(final Map<String, String> attributes, final long size) {
- this.attributes = attributes;
- this.size = size;
- }
-
-
- @Override
- public long getId() {
- return id;
- }
-
- @Override
- public long getEntryDate() {
- return entryDate;
- }
-
- @Override
- public long getLineageStartDate() {
- return entryDate;
- }
-
- @Override
- public Long getLastQueueDate() {
- return null;
- }
-
- @Override
- public Set<String> getLineageIdentifiers() {
- return Collections.emptySet();
- }
-
- @Override
- public boolean isPenalized() {
- return false;
- }
-
- @Override
- public String getAttribute(String key) {
- return attributes.get(key);
- }
-
- @Override
- public long getSize() {
- return size;
- }
-
- @Override
- public Map<String, String> getAttributes() {
- return Collections.unmodifiableMap(attributes);
- }
-
- @Override
- public int compareTo(final FlowFile o) {
- return Long.compare(id, o.getId());
- }
-
- @Override
- public long getPenaltyExpirationMillis() {
- return 0;
- }
-
- @Override
- public ContentClaim getContentClaim() {
- return null;
- }
-
- @Override
- public long getContentClaimOffset() {
- return 0;
- }
- }
-
- private static class FlowFileSizePrioritizer implements FlowFilePrioritizer {
- @Override
- public int compare(final FlowFile o1, final FlowFile o2) {
- return Long.compare(o1.getSize(), o2.getSize());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/nifi/blob/77f7d752/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FileSystemSwapManager.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FileSystemSwapManager.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FileSystemSwapManager.java
index c4a86f2..1162f39 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FileSystemSwapManager.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FileSystemSwapManager.java
@@ -173,11 +173,6 @@ public class FileSystemSwapManager implements FlowFileSwapManager {
}
}
- @Override
- public void dropSwappedFlowFiles(final String swapLocation, final FlowFileQueue flowFileQueue, final String user) throws IOException {
-
- }
-
@Override
public List<String> recoverSwapLocations(final FlowFileQueue flowFileQueue) throws IOException {
[15/30] nifi git commit: NIFI-730: Return DropFlowFileStatus object
when calling cancel
Posted by ma...@apache.org.
NIFI-730: Return DropFlowFileStatus object when calling cancel
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/0af1acaa
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/0af1acaa
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/0af1acaa
Branch: refs/heads/master
Commit: 0af1acaafaf28844b31c41e276c4dbce18390acb
Parents: 77f7d75
Author: Mark Payne <ma...@hotmail.com>
Authored: Wed Oct 14 09:46:21 2015 -0400
Committer: Mark Payne <ma...@hotmail.com>
Committed: Wed Oct 14 09:46:21 2015 -0400
----------------------------------------------------------------------
.../nifi/controller/queue/FlowFileQueue.java | 10 ++++++----
.../nifi/controller/StandardFlowFileQueue.java | 19 ++++++++++++++-----
.../controller/TestStandardFlowFileQueue.java | 7 ++-----
3 files changed, 22 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/0af1acaa/nifi-api/src/main/java/org/apache/nifi/controller/queue/FlowFileQueue.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/queue/FlowFileQueue.java b/nifi-api/src/main/java/org/apache/nifi/controller/queue/FlowFileQueue.java
index 2d67d58..dcf7f13 100644
--- a/nifi-api/src/main/java/org/apache/nifi/controller/queue/FlowFileQueue.java
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/queue/FlowFileQueue.java
@@ -199,11 +199,13 @@ public interface FlowFileQueue {
DropFlowFileStatus getDropFlowFileStatus(String requestIdentifier);
/**
- * Cancels the request to drop FlowFiles that has the given identifier
+ * Cancels the request to drop FlowFiles that has the given identifier. After this method is called, the request
+ * will no longer be known by this queue, so subsequent calls to {@link #getDropFlowFileStatus(String)} or
+ * {@link #cancelDropFlowFileRequest(String)} will return <code>null</code>
*
* @param requestIdentifier the identifier of the Drop FlowFile Request
- * @return <code>true</code> if the request was canceled, <code>false</code> if the request has
- * already completed or is not known
+ * @return the status for the request with the given identifier after it has been canceled, or <code>null</code> if no
+ * request status exists with that identifier
*/
- boolean cancelDropFlowFileRequest(String requestIdentifier);
+ DropFlowFileStatus cancelDropFlowFileRequest(String requestIdentifier);
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/0af1acaa/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
index acf2830..8085760 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
@@ -1078,7 +1078,16 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
builder.setLineageStartDate(flowFile.getLineageStartDate());
builder.setComponentId(getIdentifier());
builder.setComponentType("Connection");
- builder.setDetails("FlowFile manually dropped; request made by " + requestor);
+ builder.setAttributes(flowFile.getAttributes(), Collections.<String, String> emptyMap());
+ builder.setDetails("Manually dropped by " + requestor);
+ builder.setSourceQueueIdentifier(getIdentifier());
+
+ final ContentClaim contentClaim = flowFile.getContentClaim();
+ if (contentClaim != null) {
+ final ResourceClaim resourceClaim = contentClaim.getResourceClaim();
+ builder.setPreviousContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), contentClaim.getOffset(), flowFile.getSize());
+ }
+
return builder.build();
}
@@ -1138,14 +1147,14 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
@Override
- public boolean cancelDropFlowFileRequest(final String requestIdentifier) {
+ public DropFlowFileRequest cancelDropFlowFileRequest(final String requestIdentifier) {
final DropFlowFileRequest request = dropRequestMap.remove(requestIdentifier);
if (request == null) {
- return false;
+ return null;
}
- final boolean successful = request.cancel();
- return successful;
+ request.cancel();
+ return request;
}
@Override
http://git-wip-us.apache.org/repos/asf/nifi/blob/0af1acaa/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestStandardFlowFileQueue.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestStandardFlowFileQueue.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestStandardFlowFileQueue.java
index 3789ea5..4b67d91 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestStandardFlowFileQueue.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestStandardFlowFileQueue.java
@@ -156,7 +156,7 @@ public class TestStandardFlowFileQueue {
queue.poll(exp);
}
- @Test
+ @Test(timeout = 20000)
public void testDropSwappedFlowFiles() {
for (int i = 1; i <= 210000; i++) {
queue.put(new TestFlowFile());
@@ -165,15 +165,12 @@ public class TestStandardFlowFileQueue {
assertEquals(20, swapManager.swappedOut.size());
final DropFlowFileStatus status = queue.dropFlowFiles("1", "Unit Test");
while (status.getState() != DropFlowFileState.COMPLETE) {
- final QueueSize queueSize = queue.size();
- System.out.println(queueSize);
try {
- Thread.sleep(1000L);
+ Thread.sleep(100L);
} catch (final Exception e) {
}
}
- System.out.println(queue.size());
assertEquals(0, queue.size().getObjectCount());
assertEquals(0, queue.size().getByteCount());
assertEquals(0, swapManager.swappedOut.size());
[23/30] nifi git commit: NIFI-730: do not interrupt swap thread if
drop flowfiles is canceled
Posted by ma...@apache.org.
NIFI-730: do not interrupt swap thread if drop flowfiles is canceled
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/2b4999c0
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/2b4999c0
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/2b4999c0
Branch: refs/heads/master
Commit: 2b4999c0185ad6009dda33badfef3ade6cb2fde6
Parents: b8dbd10
Author: Mark Payne <ma...@hotmail.com>
Authored: Fri Oct 16 09:06:13 2015 -0400
Committer: Mark Payne <ma...@hotmail.com>
Committed: Fri Oct 16 09:06:13 2015 -0400
----------------------------------------------------------------------
.../org/apache/nifi/controller/DropFlowFileRequest.java | 9 ---------
.../org/apache/nifi/controller/StandardFlowFileQueue.java | 1 -
2 files changed, 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/2b4999c0/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/DropFlowFileRequest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/DropFlowFileRequest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/DropFlowFileRequest.java
index 7eea86a..6f55e79 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/DropFlowFileRequest.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/DropFlowFileRequest.java
@@ -29,7 +29,6 @@ public class DropFlowFileRequest implements DropFlowFileStatus {
private volatile QueueSize currentSize;
private volatile QueueSize droppedSize = new QueueSize(0, 0L);
private volatile long lastUpdated = System.currentTimeMillis();
- private volatile Thread executionThread;
private volatile String failureReason;
private DropFlowFileState state = DropFlowFileState.WAITING_FOR_LOCK;
@@ -101,20 +100,12 @@ public class DropFlowFileRequest implements DropFlowFileStatus {
this.lastUpdated = System.currentTimeMillis();
}
- void setExecutionThread(final Thread thread) {
- this.executionThread = thread;
- }
-
synchronized boolean cancel() {
if (this.state == DropFlowFileState.COMPLETE || this.state == DropFlowFileState.CANCELED) {
return false;
}
this.state = DropFlowFileState.CANCELED;
- if (executionThread != null) {
- executionThread.interrupt();
- }
-
return true;
}
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/2b4999c0/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
index a9e80b0..ddb8225 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
@@ -1051,7 +1051,6 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
t.setDaemon(true);
t.start();
- dropRequest.setExecutionThread(t);
dropRequestMap.put(requestIdentifier, dropRequest);
return dropRequest;
[08/30] nifi git commit: NIFI-730: - Fixing checkstyle violations. -
Renaming context menu item for emptying a queue.
Posted by ma...@apache.org.
NIFI-730:
- Fixing checkstyle violations.
- Renaming context menu item for emptying a queue.
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/09d6fe5c
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/09d6fe5c
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/09d6fe5c
Branch: refs/heads/master
Commit: 09d6fe5cdb93ed3660263165f909e088244796db
Parents: ad6af95
Author: Matt Gilman <ma...@gmail.com>
Authored: Tue Oct 13 10:12:58 2015 -0400
Committer: Matt Gilman <ma...@gmail.com>
Committed: Tue Oct 13 10:12:58 2015 -0400
----------------------------------------------------------------------
.../java/org/apache/nifi/web/NiFiServiceFacade.java | 14 +++++++-------
.../org/apache/nifi/web/api/ConnectionResource.java | 2 +-
.../java/org/apache/nifi/web/dao/ConnectionDAO.java | 12 ++++++------
.../src/main/webapp/js/nf/canvas/nf-actions.js | 2 +-
.../src/main/webapp/js/nf/canvas/nf-context-menu.js | 2 +-
5 files changed, 16 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/09d6fe5c/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java
index 28f6b61..5f89d8e 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java
@@ -528,28 +528,28 @@ public interface NiFiServiceFacade {
/**
* Creates a new flow file drop request.
- *
+ *
* @param groupId group
* @param connectionId The ID of the connection
- * @return
+ * @return The DropRequest
*/
DropRequestDTO createFlowFileDropRequest(String groupId, String connectionId);
-
+
/**
* Gets the specified flow file drop request.
- *
+ *
* @param dropRequestId The flow file drop request
* @return The DropRequest
*/
DropRequestDTO getFlowFileDropRequest(String dropRequestId);
-
+
/**
* Cancels/removes the specified flow file drop request.
- *
+ *
* @param dropRequestId The flow file drop request
*/
void deleteFlowFileDropRequest(String dropRequestId);
-
+
// ----------------------------------------
// InputPort methods
// ----------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/09d6fe5c/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectionResource.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectionResource.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectionResource.java
index dfc20fb..4265ead 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectionResource.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectionResource.java
@@ -1036,7 +1036,7 @@ public class ConnectionResource extends ApplicationResource {
/**
* Deletes the specified drop request.
- *
+ *
* @param clientId Optional client id. If the client id is not specified, a new one will be generated. This value (whether specified or generated) is included in the response.
* @param connectionId The connection id
* @param dropRequestId The drop request id
http://git-wip-us.apache.org/repos/asf/nifi/blob/09d6fe5c/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectionDAO.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectionDAO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectionDAO.java
index ce1d1fd..8c09919 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectionDAO.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectionDAO.java
@@ -33,11 +33,11 @@ public interface ConnectionDAO {
/**
* Gets the specified flow file drop request.
- *
+ *
* @param dropRequestId The drop request id
*/
void getFlowFileDropRequest(String dropRequestId);
-
+
/**
* Gets the connections for the specified source processor.
*
@@ -75,12 +75,12 @@ public interface ConnectionDAO {
/**
* Creates a new flow file drop request.
- *
+ *
* @param groupId group id
* @param id connection id
*/
void createFileFlowDropRequest(String groupId, String id);
-
+
/**
* Verifies the create request can be processed.
*
@@ -121,10 +121,10 @@ public interface ConnectionDAO {
* @param id The id of the connection
*/
void deleteConnection(String groupId, String id);
-
+
/**
* Deletes the specified flow file drop request.
- *
+ *
* @param dropRequestId The drop request id
*/
void deleteFlowFileDropRequest(String dropRequestId);
http://git-wip-us.apache.org/repos/asf/nifi/blob/09d6fe5c/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js
index bab2236..1d8152b 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js
@@ -850,7 +850,7 @@ nf.Actions = (function () {
*
* @param {type} selection
*/
- deleteQueueContents: function (selection) {
+ emptyQueue: function (selection) {
if (selection.size() !== 1 || !nf.CanvasUtils.isConnection(selection)) {
return;
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/09d6fe5c/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-context-menu.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-context-menu.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-context-menu.js
index 58397d4..f2b358d 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-context-menu.js
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-context-menu.js
@@ -382,7 +382,7 @@ nf.ContextMenu = (function () {
{condition: isCopyable, menuItem: {img: 'images/iconCopy.png', text: 'Copy', action: 'copy'}},
{condition: isPastable, menuItem: {img: 'images/iconPaste.png', text: 'Paste', action: 'paste'}},
{condition: canMoveToParent, menuItem: {img: 'images/iconMoveToParent.png', text: 'Move to parent group', action: 'moveIntoParent'}},
- {condition: canDeleteFlowFiles, menuItem: {img: 'images/iconDelete.png', text: 'Delete Flow Files', action: 'deleteQueueContents'}},
+ {condition: canDeleteFlowFiles, menuItem: {img: 'images/iconDelete.png', text: 'Empty queue', action: 'emptyQueue'}},
{condition: isDeletable, menuItem: {img: 'images/iconDelete.png', text: 'Delete', action: 'delete'}}
];
[10/30] nifi git commit: NIFI-730: - Fixing checkstyle violations. -
Wiring endpoints and framework model. - Lowering the max delay while polling
from 8 seconds to 4 seconds.
Posted by ma...@apache.org.
NIFI-730:
- Fixing checkstyle violations.
- Wiring endpoints and framework model.
- Lowering the max delay while polling from 8 seconds to 4 seconds.
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/4b41aaab
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/4b41aaab
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/4b41aaab
Branch: refs/heads/master
Commit: 4b41aaab029518909b58bf069a0c8b2a2bdca91b
Parents: af78354
Author: Matt Gilman <ma...@gmail.com>
Authored: Tue Oct 13 13:43:10 2015 -0400
Committer: Matt Gilman <ma...@gmail.com>
Committed: Tue Oct 13 13:43:10 2015 -0400
----------------------------------------------------------------------
.../apache/nifi/controller/queue/QueueSize.java | 2 +-
.../apache/nifi/web/api/dto/DropRequestDTO.java | 140 ++++++++++++++++---
.../org/apache/nifi/web/NiFiServiceFacade.java | 11 +-
.../nifi/web/StandardNiFiServiceFacade.java | 23 +--
.../apache/nifi/web/api/ConnectionResource.java | 8 +-
.../org/apache/nifi/web/api/dto/DtoFactory.java | 47 ++++++-
.../org/apache/nifi/web/dao/ConnectionDAO.java | 14 +-
.../web/dao/impl/StandardConnectionDAO.java | 35 ++++-
.../src/main/webapp/js/nf/canvas/nf-actions.js | 6 +-
9 files changed, 231 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/4b41aaab/nifi-api/src/main/java/org/apache/nifi/controller/queue/QueueSize.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/queue/QueueSize.java b/nifi-api/src/main/java/org/apache/nifi/controller/queue/QueueSize.java
index 7998d37..a59db4c 100644
--- a/nifi-api/src/main/java/org/apache/nifi/controller/queue/QueueSize.java
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/queue/QueueSize.java
@@ -50,7 +50,7 @@ public class QueueSize {
/**
* Returns a new QueueSize that is the sum of this QueueSize and the provided QueueSize
- *
+ *
* @param other the other QueueSize to add to this QueueSize
* @return a new QueueSize that is the sum of this QueueSize and the provided QueueSize
*/
http://git-wip-us.apache.org/repos/asf/nifi/blob/4b41aaab/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/DropRequestDTO.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/DropRequestDTO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/DropRequestDTO.java
index dd4289f..f022469 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/DropRequestDTO.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/DropRequestDTO.java
@@ -32,11 +32,20 @@ public class DropRequestDTO {
private String uri;
private Date submissionTime;
- private Date expiration;
+ private Date lastUpdated;
private Integer percentCompleted;
private Boolean finished;
+ private Integer currentCount;
+ private String currentSize;
+ private Integer originalCount;
+ private String originalSize;
+ private Integer droppedCount;
+ private String droppedSize;
+
+ private String state;
+
/**
* The id for this component.
*
@@ -85,21 +94,6 @@ public class DropRequestDTO {
}
/**
- * @return expiration time of the query results
- */
- @XmlJavaTypeAdapter(TimestampAdapter.class)
- @ApiModelProperty(
- value = "The timestamp when the query will expire."
- )
- public Date getExpiration() {
- return expiration;
- }
-
- public void setExpiration(Date expiration) {
- this.expiration = expiration;
- }
-
- /**
* @return percent completed
*/
@ApiModelProperty(
@@ -126,4 +120,118 @@ public class DropRequestDTO {
public void setFinished(Boolean finished) {
this.finished = finished;
}
+
+ /**
+ * @return the time this request was last updated
+ */
+ @XmlJavaTypeAdapter(TimestampAdapter.class)
+ @ApiModelProperty(
+ value = "The last time this drop request was updated."
+ )
+ public Date getLastUpdated() {
+ return lastUpdated;
+ }
+
+ public void setLastUpdated(Date lastUpdated) {
+ this.lastUpdated = lastUpdated;
+ }
+
+ /**
+ * @return the number of flow files currently queued.
+ */
+ @ApiModelProperty(
+ value = "The number of flow files currently queued."
+ )
+ public Integer getCurrentCount() {
+ return currentCount;
+ }
+
+ public void setCurrentCount(Integer currentCount) {
+ this.currentCount = currentCount;
+ }
+
+ /**
+ * @return the siez of the flow files currently queued.
+ */
+ @ApiModelProperty(
+ value = "The size of flow files currently queued."
+ )
+ public String getCurrentSize() {
+ return currentSize;
+ }
+
+ public void setCurrentSize(String currentSize) {
+ this.currentSize = currentSize;
+ }
+
+ /**
+ * @return the number of flow files to be dropped as a result of this request.
+ */
+ @ApiModelProperty(
+ value = "The number of flow files to be dropped as a result of this request."
+ )
+ public Integer getOriginalCount() {
+ return originalCount;
+ }
+
+ public void setOriginalCount(Integer originalCount) {
+ this.originalCount = originalCount;
+ }
+
+ /**
+ * @return the size of the flow files to be dropped as a result of this request.
+ */
+ @ApiModelProperty(
+ value = "The size of flow files to be dropped as a result of this request."
+ )
+ public String getOriginalSize() {
+ return originalSize;
+ }
+
+ public void setOriginalSize(String originalSize) {
+ this.originalSize = originalSize;
+ }
+
+ /**
+ * @return the number of flow files that have been dropped thus far.
+ */
+ @ApiModelProperty(
+ value = "The number of flow files that have been dropped thus far."
+ )
+ public Integer getDroppedCount() {
+ return droppedCount;
+ }
+
+ public void setDroppedCount(Integer droppedCount) {
+ this.droppedCount = droppedCount;
+ }
+
+ /**
+ * @return the size of the flow files that have been dropped thus far.
+ */
+ @ApiModelProperty(
+ value = "The size of flow files that have been dropped thus far."
+ )
+ public String getDroppedSize() {
+ return droppedSize;
+ }
+
+ public void setDroppedSize(String droppedSize) {
+ this.droppedSize = droppedSize;
+ }
+
+ /**
+ * @return the current state of the drop request.
+ */
+ @ApiModelProperty(
+ value = "The current state of the drop request."
+ )
+ public String getState() {
+ return state;
+ }
+
+ public void setState(String state) {
+ this.state = state;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/4b41aaab/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java
index 5f89d8e..0703a41 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java
@@ -531,24 +531,29 @@ public interface NiFiServiceFacade {
*
* @param groupId group
* @param connectionId The ID of the connection
+ * @param dropRequestId The ID of the drop request
* @return The DropRequest
*/
- DropRequestDTO createFlowFileDropRequest(String groupId, String connectionId);
+ DropRequestDTO createFlowFileDropRequest(String groupId, String connectionId, String dropRequestId);
/**
* Gets the specified flow file drop request.
*
+ * @param groupId group
+ * @param connectionId The ID of the connection
* @param dropRequestId The flow file drop request
* @return The DropRequest
*/
- DropRequestDTO getFlowFileDropRequest(String dropRequestId);
+ DropRequestDTO getFlowFileDropRequest(String groupId, String connectionId, String dropRequestId);
/**
* Cancels/removes the specified flow file drop request.
*
+ * @param groupId group
+ * @param connectionId The ID of the connection
* @param dropRequestId The flow file drop request
*/
- void deleteFlowFileDropRequest(String dropRequestId);
+ void deleteFlowFileDropRequest(String groupId, String connectionId, String dropRequestId);
// ----------------------------------------
// InputPort methods
http://git-wip-us.apache.org/repos/asf/nifi/blob/4b41aaab/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java
index 7f0a296..b2e6000 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java
@@ -810,8 +810,8 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade {
}
@Override
- public void deleteFlowFileDropRequest(String dropRequestId) {
- // TODO
+ public void deleteFlowFileDropRequest(String groupId, String connectionId, String dropRequestId) {
+ connectionDAO.deleteFlowFileDropRequest(groupId, connectionId, dropRequestId);
}
@Override
@@ -1066,15 +1066,8 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade {
}
@Override
- public DropRequestDTO createFlowFileDropRequest(String groupId, String connectionId) {
- // TODO
- final DropRequestDTO dto = new DropRequestDTO();
- dto.setFinished(false);
- dto.setSubmissionTime(new Date());
- dto.setExpiration(new Date(System.currentTimeMillis() + 10000));
- dto.setId(UUID.randomUUID().toString());
- dto.setPercentCompleted(100);
- return dto;
+ public DropRequestDTO createFlowFileDropRequest(String groupId, String connectionId, String dropRequestId) {
+ return dtoFactory.createDropRequestDTO(connectionDAO.createFileFlowDropRequest(groupId, connectionId, dropRequestId));
}
@Override
@@ -2110,9 +2103,8 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade {
}
@Override
- public DropRequestDTO getFlowFileDropRequest(String dropRequestId) {
- // TODO
- return null;
+ public DropRequestDTO getFlowFileDropRequest(String groupId, String connectionId, String dropRequestId) {
+ return dtoFactory.createDropRequestDTO(connectionDAO.getFlowFileDropRequest(groupId, connectionId, dropRequestId));
}
@Override
@@ -3460,8 +3452,7 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade {
}
/**
- * Utility method for extracting component counts from the specified group
- * status.
+ * Utility method for extracting component counts from the specified group status.
*/
private ProcessGroupCounts extractProcessGroupCounts(ProcessGroupStatus groupStatus) {
int running = 0;
http://git-wip-us.apache.org/repos/asf/nifi/blob/4b41aaab/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectionResource.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectionResource.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectionResource.java
index 4265ead..0d00468 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectionResource.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectionResource.java
@@ -946,8 +946,8 @@ public class ConnectionResource extends ApplicationResource {
}
// submit the drop request
- final DropRequestDTO dropRequest = serviceFacade.createFlowFileDropRequest(groupId, id);
- dropRequest.setUri(generateResourceUri("controller", "process-groups", groupId, "connections", id, "contents", "drop-requests", dropRequestId));
+ final DropRequestDTO dropRequest = serviceFacade.createFlowFileDropRequest(groupId, id, dropRequestId);
+ dropRequest.setUri(generateResourceUri("controller", "process-groups", groupId, "connections", id, "contents", "drop-requests", dropRequest.getId()));
// create the revision
final RevisionDTO revision = new RevisionDTO();
@@ -1019,7 +1019,7 @@ public class ConnectionResource extends ApplicationResource {
}
// get the drop request
- final DropRequestDTO dropRequest = serviceFacade.getFlowFileDropRequest(dropRequestId);
+ final DropRequestDTO dropRequest = serviceFacade.getFlowFileDropRequest(groupId, connectionId, dropRequestId);
dropRequest.setUri(generateResourceUri("controller", "process-groups", groupId, "connections", connectionId, "contents", "drop-requests", dropRequestId));
// create the revision
@@ -1086,7 +1086,7 @@ public class ConnectionResource extends ApplicationResource {
}
// delete the drop request
- serviceFacade.deleteFlowFileDropRequest(dropRequestId);
+ serviceFacade.deleteFlowFileDropRequest(groupId, connectionId, dropRequestId);
// create the revision
final RevisionDTO revision = new RevisionDTO();
http://git-wip-us.apache.org/repos/asf/nifi/blob/4b41aaab/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
index 76bce6f..d123704 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
@@ -124,6 +124,9 @@ import org.apache.nifi.web.api.dto.status.StatusDTO;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.controller.ConfiguredComponent;
import org.apache.nifi.controller.ReportingTaskNode;
+import org.apache.nifi.controller.queue.DropFlowFileState;
+import org.apache.nifi.controller.queue.DropFlowFileStatus;
+import org.apache.nifi.controller.queue.QueueSize;
import org.apache.nifi.controller.service.ControllerServiceNode;
import org.apache.nifi.controller.service.ControllerServiceReference;
import org.apache.nifi.reporting.ReportingTask;
@@ -294,6 +297,42 @@ public final class DtoFactory {
}
/**
+ * Creates a DropRequestDTO from the specified flow file status.
+ *
+ * @param dropRequest dropRequest
+ * @return dto
+ */
+ public DropRequestDTO createDropRequestDTO(final DropFlowFileStatus dropRequest) {
+ final DropRequestDTO dto = new DropRequestDTO();
+ dto.setId(dropRequest.getRequestIdentifier());
+ dto.setSubmissionTime(new Date(dropRequest.getRequestSubmissionTime()));
+ dto.setLastUpdated(new Date(dropRequest.getLastUpdated()));
+ dto.setState(dropRequest.getState().toString());
+ dto.setFinished(DropFlowFileState.COMPLETE.equals(dropRequest.getState()));
+
+ final QueueSize dropped = dropRequest.getDroppedSize();
+ dto.setDroppedCount(dropped.getObjectCount());
+ dto.setDroppedSize(FormatUtils.formatDataSize(dropped.getByteCount()));
+
+ if (dropRequest.getOriginalSize() != null) {
+ final QueueSize original = dropRequest.getOriginalSize();
+ dto.setOriginalCount(original.getObjectCount());
+ dto.setOriginalSize(FormatUtils.formatDataSize(original.getByteCount()));
+ dto.setPercentCompleted((dropped.getObjectCount() * 100 ) / original.getObjectCount());
+ } else {
+ dto.setPercentCompleted(0);
+ }
+
+ if (dropRequest.getCurrentSize() != null) {
+ final QueueSize current = dropRequest.getCurrentSize();
+ dto.setCurrentCount(current.getObjectCount());
+ dto.setCurrentSize(FormatUtils.formatDataSize(current.getByteCount()));
+ }
+
+ return dto;
+ }
+
+ /**
* Creates a ConnectionDTO from the specified Connection.
*
* @param connection connection
@@ -1521,8 +1560,7 @@ public final class DtoFactory {
}
/**
- * Creates a ProvenanceEventNodeDTO for the specified
- * ProvenanceEventLineageNode.
+ * Creates a ProvenanceEventNodeDTO for the specified ProvenanceEventLineageNode.
*
* @param node node
* @return dto
@@ -2173,9 +2211,8 @@ public final class DtoFactory {
/**
*
* @param original orig
- * @param deep if <code>true</code>, all Connections, ProcessGroups, Ports,
- * Processors, etc. will be copied. If <code>false</code>, the copy will
- * have links to the same objects referenced by <code>original</code>.
+ * @param deep if <code>true</code>, all Connections, ProcessGroups, Ports, Processors, etc. will be copied. If <code>false</code>, the copy will have links to the same objects referenced by
+ * <code>original</code>.
*
* @return dto
*/
http://git-wip-us.apache.org/repos/asf/nifi/blob/4b41aaab/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectionDAO.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectionDAO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectionDAO.java
index 8c09919..3665a4e 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectionDAO.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectionDAO.java
@@ -18,6 +18,7 @@ package org.apache.nifi.web.dao;
import java.util.Set;
import org.apache.nifi.connectable.Connection;
+import org.apache.nifi.controller.queue.DropFlowFileStatus;
import org.apache.nifi.web.api.dto.ConnectionDTO;
public interface ConnectionDAO {
@@ -34,9 +35,12 @@ public interface ConnectionDAO {
/**
* Gets the specified flow file drop request.
*
+ * @param groupId group id
+ * @param id The id of the connection
* @param dropRequestId The drop request id
+ * @return The drop request status
*/
- void getFlowFileDropRequest(String dropRequestId);
+ DropFlowFileStatus getFlowFileDropRequest(String groupId, String id, String dropRequestId);
/**
* Gets the connections for the specified source processor.
@@ -78,8 +82,10 @@ public interface ConnectionDAO {
*
* @param groupId group id
* @param id connection id
+ * @param dropRequestId drop request id
+ * @return The drop request status
*/
- void createFileFlowDropRequest(String groupId, String id);
+ DropFlowFileStatus createFileFlowDropRequest(String groupId, String id, String dropRequestId);
/**
* Verifies the create request can be processed.
@@ -125,7 +131,9 @@ public interface ConnectionDAO {
/**
* Deletes the specified flow file drop request.
*
+ * @param groupId group id
+ * @param id The id of the connection
* @param dropRequestId The drop request id
*/
- void deleteFlowFileDropRequest(String dropRequestId);
+ void deleteFlowFileDropRequest(String groupId, String id, String dropRequestId);
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/4b41aaab/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java
index 8fa9d3b..399ffa1 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java
@@ -23,6 +23,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
+import javax.ws.rs.WebApplicationException;
import org.apache.nifi.connectable.Connectable;
import org.apache.nifi.connectable.ConnectableType;
@@ -31,17 +32,21 @@ import org.apache.nifi.connectable.Position;
import org.apache.nifi.controller.FlowController;
import org.apache.nifi.controller.ProcessorNode;
import org.apache.nifi.controller.exception.ValidationException;
+import org.apache.nifi.controller.queue.DropFlowFileStatus;
+import org.apache.nifi.controller.queue.FlowFileQueue;
import org.apache.nifi.groups.ProcessGroup;
import org.apache.nifi.groups.RemoteProcessGroup;
import org.apache.nifi.flowfile.FlowFilePrioritizer;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.remote.RemoteGroupPort;
+import org.apache.nifi.user.NiFiUser;
import org.apache.nifi.util.FormatUtils;
import org.apache.nifi.web.ResourceNotFoundException;
import org.apache.nifi.web.api.dto.ConnectableDTO;
import org.apache.nifi.web.api.dto.ConnectionDTO;
import org.apache.nifi.web.api.dto.PositionDTO;
import org.apache.nifi.web.dao.ConnectionDAO;
+import org.apache.nifi.web.security.user.NiFiUserUtils;
public class StandardConnectionDAO extends ComponentDAO implements ConnectionDAO {
@@ -69,8 +74,16 @@ public class StandardConnectionDAO extends ComponentDAO implements ConnectionDAO
}
@Override
- public void getFlowFileDropRequest(String dropRequestId) {
- // TODO
+ public DropFlowFileStatus getFlowFileDropRequest(String groupId, String connectionId, String dropRequestId) {
+ final Connection connection = locateConnection(groupId, connectionId);
+ final FlowFileQueue queue = connection.getFlowFileQueue();
+
+ final DropFlowFileStatus dropRequest = queue.getDropFlowFileStatus(dropRequestId);
+ if (dropRequest == null) {
+ throw new ResourceNotFoundException(String.format("Unable to find drop request with id '%s'.", dropRequestId));
+ }
+
+ return dropRequest;
}
@Override
@@ -299,8 +312,16 @@ public class StandardConnectionDAO extends ComponentDAO implements ConnectionDAO
}
@Override
- public void createFileFlowDropRequest(String groupId, String id) {
- // TODO
+ public DropFlowFileStatus createFileFlowDropRequest(String groupId, String id, String dropRequestId) {
+ final Connection connection = locateConnection(groupId, id);
+ final FlowFileQueue queue = connection.getFlowFileQueue();
+
+ final NiFiUser user = NiFiUserUtils.getNiFiUser();
+ if (user == null) {
+ throw new WebApplicationException(new Throwable("Unable to access details for current user."));
+ }
+
+ return queue.dropFlowFiles(dropRequestId, user.getDn());
}
@Override
@@ -475,8 +496,10 @@ public class StandardConnectionDAO extends ComponentDAO implements ConnectionDAO
}
@Override
- public void deleteFlowFileDropRequest(String dropRequestId) {
- // TODO
+ public void deleteFlowFileDropRequest(String groupId, String connectionId, String dropRequestId) {
+ final Connection connection = locateConnection(groupId, connectionId);
+ final FlowFileQueue queue = connection.getFlowFileQueue();
+ queue.cancelDropFlowFileRequest(dropRequestId);
}
/* setters */
http://git-wip-us.apache.org/repos/asf/nifi/blob/4b41aaab/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js
index 1d8152b..2a94ec5 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js
@@ -854,13 +854,17 @@ nf.Actions = (function () {
if (selection.size() !== 1 || !nf.CanvasUtils.isConnection(selection)) {
return;
}
+
+ var MAX_DELAY = 4;
// process the drop request
var processDropRequest = function (dropRequest, nextDelay) {
// see if the drop request has completed
if (dropRequest.finished === true) {
+ // request is finished so it can be removed
deleteDropRequest(dropRequest);
} else {
+ // update the UI with the current status of the drop request
schedule(dropRequest, nextDelay);
}
};
@@ -874,7 +878,7 @@ nf.Actions = (function () {
dataType: 'json'
}).done(function(response) {
var dropRequest = response.dropRequest;
- processDropRequest(dropRequest, Math.min(8, delay * 2));
+ processDropRequest(dropRequest, Math.min(MAX_DELAY, delay * 2));
}).fail(nf.Common.handleAjaxError);
}, delay * 1000);
};
[29/30] nifi git commit: Merge branch 'master' into NIFI-730
Posted by ma...@apache.org.
Merge branch 'master' into NIFI-730
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/e4cebba3
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/e4cebba3
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/e4cebba3
Branch: refs/heads/master
Commit: e4cebba3c7868010c2b9fd994b850f40b81aa044
Parents: 5a04021 a549621
Author: Mark Payne <ma...@hotmail.com>
Authored: Mon Nov 2 14:02:16 2015 -0500
Committer: Mark Payne <ma...@hotmail.com>
Committed: Mon Nov 2 14:02:16 2015 -0500
----------------------------------------------------------------------
.../java/org/apache/nifi/action/Action.java | 50 ++
.../java/org/apache/nifi/action/Component.java | 34 +
.../java/org/apache/nifi/action/Operation.java | 37 +
.../component/details/ComponentDetails.java | 26 +
.../component/details/ExtensionDetails.java | 26 +
.../details/RemoteProcessGroupDetails.java | 26 +
.../nifi/action/details/ActionDetails.java | 26 +
.../nifi/action/details/ConfigureDetails.java | 30 +
.../nifi/action/details/ConnectDetails.java | 40 +
.../apache/nifi/action/details/MoveDetails.java | 30 +
.../nifi/action/details/PurgeDetails.java | 28 +
.../annotation/behavior/InputRequirement.java | 67 ++
.../apache/nifi/processor/ProcessSession.java | 27 +
.../nifi/provenance/ProvenanceEventType.java | 31 +-
.../nifi/provenance/ProvenanceReporter.java | 37 +
.../org/apache/nifi/reporting/EventAccess.java | 13 +
.../StandardProvenanceEventRecord.java | 36 +-
.../nifi/provenance/StandardQueryResult.java | 25 +-
.../nifi/processor/util/StandardValidators.java | 10 +
.../client/socket/EndpointConnectionPool.java | 3 +-
.../src/main/asciidoc/administration-guide.adoc | 4 +-
.../src/main/asciidoc/developer-guide.adoc | 11 +
.../src/main/asciidoc/getting-started.adoc | 754 +++++++++++++++++
.../src/main/asciidoc/images/add-processor.png | Bin 31524 -> 92164 bytes
nifi-docs/src/main/asciidoc/user-guide.adoc | 2 +-
.../src/main/java/MyProcessor.java | 11 +-
.../org/apache/nifi/util/MockEventAccess.java | 27 +
.../apache/nifi/util/MockProcessSession.java | 8 +
.../nifi/util/MockProvenanceReporter.java | 35 +-
.../nifi/processors/avro/ConvertAvroToJSON.java | 70 +-
.../processors/avro/ExtractAvroMetadata.java | 29 +-
.../apache/nifi/processors/avro/SplitAvro.java | 27 +-
.../processors/avro/TestConvertAvroToJSON.java | 47 +-
.../processors/aws/AbstractAWSProcessor.java | 2 +-
.../nifi/processors/aws/s3/DeleteS3Object.java | 98 +++
.../nifi/processors/aws/s3/FetchS3Object.java | 5 +-
.../nifi/processors/aws/s3/PutS3Object.java | 48 +-
.../apache/nifi/processors/aws/sns/PutSNS.java | 3 +
.../nifi/processors/aws/sqs/DeleteSQS.java | 3 +
.../apache/nifi/processors/aws/sqs/GetSQS.java | 5 +-
.../apache/nifi/processors/aws/sqs/PutSQS.java | 5 +-
.../org.apache.nifi.processor.Processor | 1 +
.../processors/aws/s3/TestDeleteS3Object.java | 141 ++++
.../nifi/processors/flume/ExecuteFlumeSink.java | 14 +-
.../processors/flume/ExecuteFlumeSource.java | 14 +-
.../org/apache/nifi/admin/dao/ActionDAO.java | 12 +-
.../java/org/apache/nifi/admin/dao/UserDAO.java | 3 +-
.../nifi/admin/dao/impl/StandardActionDAO.java | 93 ++-
.../nifi/admin/dao/impl/StandardUserDAO.java | 4 +-
.../apache/nifi/admin/service/AuditService.java | 18 +-
.../admin/service/action/AddActionsAction.java | 3 +-
.../service/action/PurgeActionsAction.java | 3 +-
.../service/impl/StandardAuditService.java | 24 +-
.../resources/nifi-administration-context.xml | 2 +-
.../apache/nifi/web/api/dto/ProcessorDTO.java | 15 +
.../manager/impl/ClusteredEventAccess.java | 21 +-
.../cluster/manager/impl/WebClusterManager.java | 2 +-
.../apache/nifi/controller/ProcessorNode.java | 7 +-
.../apache/nifi/controller/FlowController.java | 15 +-
.../nifi/controller/FlowUnmarshaller.java | 77 --
.../nifi/controller/StandardProcessorNode.java | 109 ++-
.../apache/nifi/controller/TemplateManager.java | 1 +
.../repository/BatchingSessionFactory.java | 5 +
.../repository/FileSystemRepository.java | 40 +-
.../repository/StandardProcessSession.java | 12 +
.../repository/StandardProvenanceReporter.java | 31 +-
.../nifi/processor/SimpleProcessLogger.java | 20 +-
.../nifi/spring/FlowControllerFactoryBean.java | 8 +
.../src/main/resources/FlowConfiguration.xsd | 2 +-
.../src/main/resources/nifi-context.xml | 1 +
.../controller/StandardFlowServiceTest.java | 5 +-
.../repository/TestFileSystemRepository.java | 18 +
.../repository/TestStandardProcessSession.java | 39 +-
.../nifi/processor/TestSimpleProcessLogger.java | 101 +++
.../src/main/resources/bin/nifi.sh | 96 +--
.../nifi-framework/nifi-user-actions/pom.xml | 7 +
.../java/org/apache/nifi/action/Action.java | 121 ---
.../java/org/apache/nifi/action/Component.java | 34 -
.../apache/nifi/action/FlowChangeAction.java | 130 +++
.../java/org/apache/nifi/action/Operation.java | 37 -
.../component/details/ComponentDetails.java | 26 -
.../component/details/ExtensionDetails.java | 34 -
.../details/FlowChangeExtensionDetails.java | 35 +
.../FlowChangeRemoteProcessGroupDetails.java | 35 +
.../details/RemoteProcessGroupDetails.java | 34 -
.../nifi/action/details/ActionDetails.java | 26 -
.../nifi/action/details/ConfigureDetails.java | 52 --
.../nifi/action/details/ConnectDetails.java | 90 --
.../details/FlowChangeConfigureDetails.java | 55 ++
.../details/FlowChangeConnectDetails.java | 97 +++
.../action/details/FlowChangeMoveDetails.java | 65 ++
.../action/details/FlowChangePurgeDetails.java | 46 +
.../apache/nifi/action/details/MoveDetails.java | 61 --
.../nifi/action/details/PurgeDetails.java | 45 -
.../apache/nifi/audit/ControllerAuditor.java | 27 +-
.../nifi/audit/ControllerServiceAuditor.java | 43 +-
.../org/apache/nifi/audit/FunnelAuditor.java | 7 +-
.../java/org/apache/nifi/audit/NiFiAuditor.java | 25 +-
.../java/org/apache/nifi/audit/PortAuditor.java | 27 +-
.../apache/nifi/audit/ProcessGroupAuditor.java | 25 +-
.../org/apache/nifi/audit/ProcessorAuditor.java | 25 +-
.../apache/nifi/audit/RelationshipAuditor.java | 18 +-
.../nifi/audit/RemoteProcessGroupAuditor.java | 35 +-
.../apache/nifi/audit/ReportingTaskAuditor.java | 25 +-
.../org/apache/nifi/audit/SnippetAuditor.java | 26 +-
.../nifi/web/StandardNiFiServiceFacade.java | 9 +-
.../StandardNiFiWebConfigurationContext.java | 13 +-
.../apache/nifi/web/StandardNiFiWebContext.java | 13 +-
.../nifi/web/api/ApplicationResource.java | 5 +-
.../org/apache/nifi/web/api/dto/DtoFactory.java | 21 +-
.../partials/provenance/provenance-content.jsp | 2 +
.../nifi-web-ui/src/main/webapp/css/main.css | 7 +-
.../src/main/webapp/js/nf/canvas/nf-actions.js | 20 +-
.../main/webapp/js/nf/canvas/nf-canvas-utils.js | 13 +-
.../js/nf/canvas/nf-connection-configuration.js | 8 +-
.../main/webapp/js/nf/canvas/nf-context-menu.js | 23 +-
.../js/nf/provenance/nf-provenance-table.js | 54 +-
.../webapp/js/nf/provenance/nf-provenance.js | 16 +-
.../org/apache/nifi/processors/GeoEnrichIP.java | 3 +
.../hadoop/AbstractHadoopProcessor.java | 92 +-
.../hadoop/CreateHadoopSequenceFile.java | 4 +-
.../nifi/processors/hadoop/FetchHDFS.java | 5 +-
.../apache/nifi/processors/hadoop/GetHDFS.java | 3 +
.../apache/nifi/processors/hadoop/ListHDFS.java | 3 +
.../apache/nifi/processors/hadoop/PutHDFS.java | 3 +
.../processors/hl7/ExtractHL7Attributes.java | 3 +
.../apache/nifi/processors/hl7/RouteHL7.java | 3 +
.../processors/image/ExtractImageMetadata.java | 36 +-
.../nifi/processors/image/ResizeImage.java | 38 +-
.../apache/nifi/processors/kafka/GetKafka.java | 21 +-
.../apache/nifi/processors/kafka/PutKafka.java | 16 +-
.../processors/kite/AbstractKiteProcessor.java | 11 +-
.../nifi/processors/kite/ConvertCSVToAvro.java | 312 +++----
.../nifi/processors/kite/ConvertJSONToAvro.java | 14 +-
.../processors/kite/StoreInKiteDataset.java | 9 +-
.../processors/kite/TestCSVToAvroProcessor.java | 39 +
.../nifi/processors/yandex/YandexTranslate.java | 3 +
.../nifi/provenance/IndexConfiguration.java | 47 +-
.../PersistentProvenanceRepository.java | 124 +--
.../nifi/provenance/lucene/DocsReader.java | 49 +-
.../nifi/provenance/lucene/IndexSearch.java | 35 +-
.../nifi/provenance/lucene/LineageQuery.java | 7 +-
.../TestPersistentProvenanceRepository.java | 163 +++-
.../VolatileProvenanceRepository.java | 16 +-
.../nifi/processors/twitter/GetTwitter.java | 5 +-
.../apache/nifi/processors/solr/GetSolr.java | 43 +-
.../processors/solr/PutSolrContentStream.java | 33 +-
.../nifi-standard-prioritizers/pom.xml | 4 +
.../PriorityAttributePrioritizer.java | 7 +-
.../PriorityAttributePrioritizerTest.java | 17 +-
.../nifi-standard-processors/pom.xml | 8 +
.../standard/AbstractListProcessor.java | 507 +++++++++++
.../processors/standard/AttributesToJSON.java | 242 ++++++
.../standard/Base64EncodeContent.java | 31 +-
.../processors/standard/CompressContent.java | 15 +-
.../nifi/processors/standard/ControlRate.java | 113 +--
.../standard/ConvertCharacterSet.java | 22 +-
.../processors/standard/ConvertJSONToSQL.java | 3 +
.../processors/standard/DetectDuplicate.java | 3 +
.../processors/standard/DistributeLoad.java | 3 +
.../processors/standard/DuplicateFlowFile.java | 3 +
.../nifi/processors/standard/EncodeContent.java | 15 +-
.../processors/standard/EncryptContent.java | 3 +
.../processors/standard/EvaluateJsonPath.java | 38 +-
.../nifi/processors/standard/EvaluateXPath.java | 29 +-
.../processors/standard/EvaluateXQuery.java | 25 +-
.../processors/standard/ExecuteProcess.java | 3 +
.../nifi/processors/standard/ExecuteSQL.java | 12 +-
.../standard/ExecuteStreamCommand.java | 7 +-
.../nifi/processors/standard/ExtractText.java | 3 +
.../processors/standard/FetchFileTransfer.java | 346 ++++++++
.../nifi/processors/standard/FetchSFTP.java | 93 +++
.../processors/standard/GenerateFlowFile.java | 11 +-
.../apache/nifi/processors/standard/GetFTP.java | 13 +-
.../nifi/processors/standard/GetFile.java | 7 +-
.../nifi/processors/standard/GetHTTP.java | 3 +
.../nifi/processors/standard/GetJMSQueue.java | 3 +
.../nifi/processors/standard/GetJMSTopic.java | 3 +
.../nifi/processors/standard/GetSFTP.java | 7 +-
.../processors/standard/HandleHttpRequest.java | 30 +-
.../processors/standard/HandleHttpResponse.java | 5 +-
.../nifi/processors/standard/HashAttribute.java | 5 +-
.../nifi/processors/standard/HashContent.java | 5 +-
.../processors/standard/IdentifyMimeType.java | 5 +-
.../nifi/processors/standard/InvokeHTTP.java | 201 ++---
.../nifi/processors/standard/JmsConsumer.java | 2 +
.../processors/standard/ListFileTransfer.java | 119 +++
.../nifi/processors/standard/ListSFTP.java | 88 ++
.../nifi/processors/standard/ListenHTTP.java | 121 +--
.../nifi/processors/standard/ListenUDP.java | 18 +-
.../nifi/processors/standard/LogAttribute.java | 16 +-
.../nifi/processors/standard/MergeContent.java | 17 +-
.../nifi/processors/standard/ModifyBytes.java | 14 +-
.../processors/standard/MonitorActivity.java | 31 +-
.../nifi/processors/standard/PostHTTP.java | 3 +
.../standard/PutDistributedMapCache.java | 252 ++++++
.../nifi/processors/standard/PutEmail.java | 3 +
.../apache/nifi/processors/standard/PutFTP.java | 3 +
.../nifi/processors/standard/PutFile.java | 3 +
.../apache/nifi/processors/standard/PutJMS.java | 7 +-
.../nifi/processors/standard/PutSFTP.java | 3 +
.../apache/nifi/processors/standard/PutSQL.java | 159 ++--
.../nifi/processors/standard/ReplaceText.java | 485 ++++++++---
.../standard/ReplaceTextWithMapping.java | 18 +-
.../processors/standard/RouteOnAttribute.java | 3 +
.../processors/standard/RouteOnContent.java | 19 +-
.../nifi/processors/standard/ScanAttribute.java | 19 +-
.../nifi/processors/standard/ScanContent.java | 5 +-
.../processors/standard/SegmentContent.java | 7 +-
.../nifi/processors/standard/SplitContent.java | 7 +-
.../nifi/processors/standard/SplitJson.java | 32 +-
.../nifi/processors/standard/SplitText.java | 53 +-
.../nifi/processors/standard/SplitXml.java | 18 +-
.../nifi/processors/standard/TransformXml.java | 3 +
.../nifi/processors/standard/UnpackContent.java | 9 +-
.../nifi/processors/standard/ValidateXml.java | 16 +-
.../servlets/ContentAcknowledgmentServlet.java | 3 +-
.../standard/servlets/ListenHTTPServlet.java | 8 +-
.../processors/standard/util/EntityListing.java | 71 ++
.../processors/standard/util/FTPTransfer.java | 145 ++--
.../nifi/processors/standard/util/FileInfo.java | 18 +-
.../processors/standard/util/FileTransfer.java | 337 ++++----
.../processors/standard/util/JdbcCommon.java | 76 +-
.../processors/standard/util/JmsFactory.java | 79 +-
.../processors/standard/util/JmsProperties.java | 8 +
.../standard/util/ListableEntity.java | 40 +
.../util/PermissionDeniedException.java | 32 +
.../processors/standard/util/SFTPTransfer.java | 191 +++--
.../org.apache.nifi.processor.Processor | 4 +
.../standard/TestAbstractListProcessor.java | 221 +++++
.../standard/TestAttributesToJSON.java | 282 +++++++
.../standard/TestConvertCharacterSet.java | 64 +-
.../standard/TestFetchFileTransfer.java | 317 +++++++
.../nifi/processors/standard/TestGetFile.java | 21 +-
.../processors/standard/TestInvokeHTTP.java | 607 +-------------
.../processors/standard/TestInvokeHttpSSL.java | 90 ++
.../standard/TestPutDistributedMapCache.java | 277 +++++++
.../nifi/processors/standard/TestPutSQL.java | 46 +-
.../processors/standard/TestReplaceText.java | 635 +++++++++++++-
.../standard/TestReplaceTextLineByLine.java | 336 --------
.../standard/util/TestInvokeHttpCommon.java | 830 +++++++++++++++++++
.../standard/util/TestJdbcCommon.java | 87 ++
.../standard/util/TestJdbcTypesDerby.java | 133 +++
.../standard/util/TestJdbcTypesH2.java | 149 ++++
.../AppendLineByLineTest.txt | 11 +
.../PrependLineByLineTest.txt | 11 +
.../processors/attributes/UpdateAttribute.java | 9 +-
pom.xml | 2 +-
248 files changed, 10506 insertions(+), 3447 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/e4cebba3/nifi-api/src/main/java/org/apache/nifi/processor/ProcessSession.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/e4cebba3/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/e4cebba3/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/impl/WebClusterManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/e4cebba3/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java
----------------------------------------------------------------------
diff --cc nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java
index 20f2642,135025e..701459c
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java
@@@ -16,40 -16,11 +16,39 @@@
*/
package org.apache.nifi.controller;
-import com.sun.jersey.api.client.ClientHandlerException;
-import org.apache.commons.lang3.StringUtils;
+import static java.util.Objects.requireNonNull;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import javax.net.ssl.SSLContext;
+
- import org.apache.commons.lang3.StringUtils;
- import org.apache.nifi.admin.service.UserService;
+ import org.apache.nifi.action.Action;
-import org.apache.nifi.admin.service.AuditService;
-import org.apache.nifi.admin.service.UserService;
import org.apache.nifi.annotation.lifecycle.OnAdded;
import org.apache.nifi.annotation.lifecycle.OnRemoved;
import org.apache.nifi.annotation.lifecycle.OnShutdown;
http://git-wip-us.apache.org/repos/asf/nifi/blob/e4cebba3/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/BatchingSessionFactory.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/e4cebba3/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/e4cebba3/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/TestStandardProcessSession.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/e4cebba3/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/e4cebba3/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/e4cebba3/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/e4cebba3/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-context-menu.js
----------------------------------------------------------------------
[19/30] nifi git commit: NIFI-730: Set current queue size on drop
flowfile request immediately when creating request
Posted by ma...@apache.org.
NIFI-730: Set current queue size on drop flowfile request immediately when creating request
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/98a04eec
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/98a04eec
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/98a04eec
Branch: refs/heads/master
Commit: 98a04eec741d5ed72405c6549970de5f1281239e
Parents: cad0e7c
Author: Mark Payne <ma...@hotmail.com>
Authored: Wed Oct 14 13:48:59 2015 -0400
Committer: Mark Payne <ma...@hotmail.com>
Committed: Wed Oct 14 13:48:59 2015 -0400
----------------------------------------------------------------------
.../main/java/org/apache/nifi/controller/StandardFlowFileQueue.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/98a04eec/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
index 062c424..b0c31e7 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
@@ -938,6 +938,7 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
}
final DropFlowFileRequest dropRequest = new DropFlowFileRequest(requestIdentifier);
+ dropRequest.setCurrentSize(size());
final Thread t = new Thread(new Runnable() {
@Override
public void run() {
[05/30] nifi git commit: NIFI-730: Implemented swapping in and out
on-demand by the FlowFileQueue rather than in a background thread
Posted by ma...@apache.org.
NIFI-730: Implemented swapping in and out on-demand by the FlowFileQueue rather than in a background thread
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/49a781df
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/49a781df
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/49a781df
Branch: refs/heads/master
Commit: 49a781df2d44859ec59672c2755b7346452cd74a
Parents: b8c51dc
Author: Mark Payne <ma...@hotmail.com>
Authored: Mon Oct 12 13:27:07 2015 -0400
Committer: Mark Payne <ma...@hotmail.com>
Committed: Tue Oct 13 10:03:03 2015 -0400
----------------------------------------------------------------------
.../controller/queue/DropFlowFileState.java | 3 +-
.../controller/queue/DropFlowFileStatus.java | 7 +-
.../nifi/controller/queue/FlowFileQueue.java | 50 --
.../apache/nifi/controller/queue/QueueSize.java | 7 +
.../repository/FlowFileSwapManager.java | 17 +-
.../SwapManagerInitializationContext.java | 1 -
.../java/org/apache/nifi/util/MockFlowFile.java | 24 +-
.../org/apache/nifi/util/MockFlowFileQueue.java | 3 +-
.../apache/nifi/util/MockProcessSession.java | 15 +-
.../nifi/util/StandardProcessorTestRunner.java | 24 +-
.../java/org/apache/nifi/util/TestRunner.java | 124 ++--
.../nifi/controller/DropFlowFileRequest.java | 99 +++
.../nifi/controller/StandardFlowFileQueue.java | 687 +++++++++----------
.../controller/TestStandardFlowFileQueue.java | 330 +++++++++
.../nifi/connectable/StandardConnection.java | 33 +-
.../nifi/controller/FileSystemSwapManager.java | 54 +-
.../apache/nifi/controller/FlowController.java | 24 +-
.../repository/StandardProcessSession.java | 102 +--
.../java/org/apache/nifi/util/Connectables.java | 2 +-
.../controller/TestFileSystemSwapManager.java | 142 +++-
.../repository/TestStandardProcessSession.java | 2 +-
.../nifi/web/controller/ControllerFacade.java | 30 +-
22 files changed, 1155 insertions(+), 625 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/49a781df/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileState.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileState.java b/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileState.java
index 3f16d00..e412b80 100644
--- a/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileState.java
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileState.java
@@ -25,7 +25,8 @@ public enum DropFlowFileState {
WAITING_FOR_LOCK("Waiting for Destination Component to complete its action"),
DROPPING_ACTIVE_FLOWFILES("Dropping FlowFiles from queue"),
COMPLETE("Completed Successfully"),
- FAILURE("Failed");
+ FAILURE("Failed"),
+ CANCELED("Cancelled by User");
private final String description;
http://git-wip-us.apache.org/repos/asf/nifi/blob/49a781df/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileStatus.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileStatus.java b/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileStatus.java
index b216608..3c3be9b 100644
--- a/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileStatus.java
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileStatus.java
@@ -43,6 +43,12 @@ public interface DropFlowFileStatus {
long getRequestSubmissionTime();
/**
+ * @return the date/time (in milliseconds since epoch) at which the status of the
+ * request was last updated
+ */
+ long getLastUpdated();
+
+ /**
* @return the size of the queue when the drop request was issued or <code>null</code> if
* it is not yet known, which can happen if the {@link DropFlowFileState} is
* {@link DropFlowFileState#WAITING_FOR_LOCK}.
@@ -58,5 +64,4 @@ public interface DropFlowFileStatus {
* @return the current state of the operation
*/
DropFlowFileState getState();
-
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/49a781df/nifi-api/src/main/java/org/apache/nifi/controller/queue/FlowFileQueue.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/queue/FlowFileQueue.java b/nifi-api/src/main/java/org/apache/nifi/controller/queue/FlowFileQueue.java
index 31f17e0..bc2f358 100644
--- a/nifi-api/src/main/java/org/apache/nifi/controller/queue/FlowFileQueue.java
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/queue/FlowFileQueue.java
@@ -60,13 +60,6 @@ public interface FlowFileQueue {
void purgeSwapFiles();
/**
- * @return the minimum number of FlowFiles that must be present in order for
- * FlowFiles to begin being swapped out of the queue
- */
- // TODO: REMOVE THIS.
- int getSwapThreshold();
-
- /**
* Resets the comparator used by this queue to maintain order.
*
* @param newPriorities the ordered list of prioritizers to use to determine
@@ -112,12 +105,8 @@ public interface FlowFileQueue {
* not include those FlowFiles that have been swapped out or are currently
* being processed
*/
- // TODO: REMOVE?
boolean isActiveQueueEmpty();
- // TODO: REMOVE?
- QueueSize getActiveQueueSize();
-
/**
* Returns a QueueSize that represents all FlowFiles that are 'unacknowledged'. A FlowFile
* is considered to be unacknowledged if it has been pulled from the queue by some component
@@ -152,45 +141,6 @@ public interface FlowFileQueue {
void putAll(Collection<FlowFileRecord> files);
/**
- * Removes all records from the internal swap queue and returns them.
- *
- * @return all removed records from internal swap queue
- */
- // TODO: REMOVE THIS?
- List<FlowFileRecord> pollSwappableRecords();
-
- /**
- * Restores the records from swap space into this queue, adding the records
- * that have expired to the given set instead of enqueuing them.
- *
- * @param records that were swapped in
- */
- // TODO: REMOVE THIS?
- void putSwappedRecords(Collection<FlowFileRecord> records);
-
- /**
- * Updates the internal counters of how much data is queued, based on
- * swapped data that is being restored.
- *
- * @param numRecords count of records swapped in
- * @param contentSize total size of records being swapped in
- */
- // TODO: REMOVE THIS?
- void incrementSwapCount(int numRecords, long contentSize);
-
- /**
- * @return the number of FlowFiles that are enqueued and not swapped
- */
- // TODO: REMOVE THIS?
- int unswappedSize();
-
- // TODO: REMOVE THIS?
- int getSwapRecordCount();
-
- // TODO: REMOVE THIS?
- int getSwapQueueSize();
-
- /**
* @param expiredRecords expired records
* @return the next flow file on the queue; null if empty
*/
http://git-wip-us.apache.org/repos/asf/nifi/blob/49a781df/nifi-api/src/main/java/org/apache/nifi/controller/queue/QueueSize.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/queue/QueueSize.java b/nifi-api/src/main/java/org/apache/nifi/controller/queue/QueueSize.java
index 42d8416..528d652 100644
--- a/nifi-api/src/main/java/org/apache/nifi/controller/queue/QueueSize.java
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/queue/QueueSize.java
@@ -16,6 +16,8 @@
*/
package org.apache.nifi.controller.queue;
+import java.text.NumberFormat;
+
/**
*
*/
@@ -45,4 +47,9 @@ public class QueueSize {
public long getByteCount() {
return totalSizeBytes;
}
+
+ @Override
+ public String toString() {
+ return "QueueSize[FlowFiles=" + objectCount + ", ContentSize=" + NumberFormat.getNumberInstance().format(totalSizeBytes) + " Bytes]";
+ }
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/49a781df/nifi-api/src/main/java/org/apache/nifi/controller/repository/FlowFileSwapManager.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/repository/FlowFileSwapManager.java b/nifi-api/src/main/java/org/apache/nifi/controller/repository/FlowFileSwapManager.java
index 57e9186..a70d287 100644
--- a/nifi-api/src/main/java/org/apache/nifi/controller/repository/FlowFileSwapManager.java
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/repository/FlowFileSwapManager.java
@@ -26,6 +26,9 @@ import org.apache.nifi.controller.queue.QueueSize;
* Defines a mechanism by which FlowFiles can be move into external storage or
* memory so that they can be removed from the Java heap and vice-versa
*/
+// TODO: This needs to be refactored into two different mechanisms, one that is responsible for doing
+// framework-y types of things, such as updating the repositories, and another that is responsible
+// for serializing and deserializing FlowFiles to external storage.
public interface FlowFileSwapManager {
/**
@@ -38,6 +41,16 @@ public interface FlowFileSwapManager {
void initialize(SwapManagerInitializationContext initializationContext);
/**
+ * Drops all FlowFiles that are swapped out at the given location. This will update the Provenance
+ * Repository as well as the FlowFile Repository and
+ *
+ * @param swapLocation the location of the swap file to drop
+ * @param flowFileQueue the queue to which the FlowFiles belong
+ * @param user the user that initiated the request
+ */
+ void dropSwappedFlowFiles(String swapLocation, FlowFileQueue flowFileQueue, String user) throws IOException;
+
+ /**
* Swaps out the given FlowFiles that belong to the queue with the given identifier.
*
* @param flowFiles the FlowFiles to swap out to external storage
@@ -53,13 +66,13 @@ public interface FlowFileSwapManager {
* provides a view of the FlowFiles but does not actively swap them in, meaning that the swap file
* at the given location remains in that location and the FlowFile Repository is not updated.
*
- * @param swapLocation the location of hte swap file
+ * @param swapLocation the location of the swap file
* @param flowFileQueue the queue that the FlowFiles belong to
* @return the FlowFiles that live at the given swap location
*
* @throws IOException if unable to recover the FlowFiles from the given location
*/
- List<FlowFileRecord> peek(String swapLocation, final FlowFileQueue flowFileQueue) throws IOException;
+ List<FlowFileRecord> peek(String swapLocation, FlowFileQueue flowFileQueue) throws IOException;
/**
* Recovers the FlowFiles from the swap file that lives at the given location and belongs
http://git-wip-us.apache.org/repos/asf/nifi/blob/49a781df/nifi-api/src/main/java/org/apache/nifi/controller/repository/SwapManagerInitializationContext.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/repository/SwapManagerInitializationContext.java b/nifi-api/src/main/java/org/apache/nifi/controller/repository/SwapManagerInitializationContext.java
index 564d5ec..0e30784 100644
--- a/nifi-api/src/main/java/org/apache/nifi/controller/repository/SwapManagerInitializationContext.java
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/repository/SwapManagerInitializationContext.java
@@ -27,7 +27,6 @@ public interface SwapManagerInitializationContext {
*/
FlowFileRepository getFlowFileRepository();
-
/**
* @return the {@link ResourceClaimManager} that is necessary to provide to the FlowFileRepository when
* performing swapping actions
http://git-wip-us.apache.org/repos/asf/nifi/blob/49a781df/nifi-mock/src/main/java/org/apache/nifi/util/MockFlowFile.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/MockFlowFile.java b/nifi-mock/src/main/java/org/apache/nifi/util/MockFlowFile.java
index e9fb9d6..41bcc74 100644
--- a/nifi-mock/src/main/java/org/apache/nifi/util/MockFlowFile.java
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/MockFlowFile.java
@@ -33,12 +33,13 @@ import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import org.apache.nifi.controller.repository.FlowFileRecord;
+import org.apache.nifi.controller.repository.claim.ContentClaim;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.flowfile.attributes.CoreAttributes;
-
import org.junit.Assert;
-public class MockFlowFile implements FlowFile {
+public class MockFlowFile implements FlowFileRecord {
private final Map<String, String> attributes = new HashMap<>();
@@ -170,7 +171,7 @@ public class MockFlowFile implements FlowFile {
public void assertAttributeNotExists(final String attributeName) {
Assert.assertFalse("Attribute " + attributeName + " not exists with value " + attributes.get(attributeName),
- attributes.containsKey(attributeName));
+ attributes.containsKey(attributeName));
}
public void assertAttributeEquals(final String attributeName, final String expectedValue) {
@@ -250,7 +251,7 @@ public class MockFlowFile implements FlowFile {
if ((fromStream & 0xFF) != (data[i] & 0xFF)) {
Assert.fail("FlowFile content differs from input at byte " + bytesRead + " with input having value "
- + (fromStream & 0xFF) + " and FlowFile having value " + (data[i] & 0xFF));
+ + (fromStream & 0xFF) + " and FlowFile having value " + (data[i] & 0xFF));
}
bytesRead++;
@@ -274,4 +275,19 @@ public class MockFlowFile implements FlowFile {
public Long getLastQueueDate() {
return entryDate;
}
+
+ @Override
+ public long getPenaltyExpirationMillis() {
+ return -1;
+ }
+
+ @Override
+ public ContentClaim getContentClaim() {
+ return null;
+ }
+
+ @Override
+ public long getContentClaimOffset() {
+ return 0;
+ }
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/49a781df/nifi-mock/src/main/java/org/apache/nifi/util/MockFlowFileQueue.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/MockFlowFileQueue.java b/nifi-mock/src/main/java/org/apache/nifi/util/MockFlowFileQueue.java
index 775a1d5..0c6ec2a 100644
--- a/nifi-mock/src/main/java/org/apache/nifi/util/MockFlowFileQueue.java
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/MockFlowFileQueue.java
@@ -23,7 +23,8 @@ import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-import org.apache.nifi.processor.QueueSize;
+import org.apache.nifi.controller.queue.QueueSize;
+
public class MockFlowFileQueue {
http://git-wip-us.apache.org/repos/asf/nifi/blob/49a781df/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java b/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java
index 1060854..85fc784 100644
--- a/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java
@@ -40,12 +40,12 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Pattern;
+import org.apache.nifi.controller.queue.QueueSize;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.flowfile.attributes.CoreAttributes;
import org.apache.nifi.processor.FlowFileFilter;
import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.Processor;
-import org.apache.nifi.processor.QueueSize;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.exception.FlowFileAccessException;
import org.apache.nifi.processor.exception.FlowFileHandlingException;
@@ -691,7 +691,7 @@ public class MockProcessSession implements ProcessSession {
/**
* @param relationship to get flowfiles for
* @return a List of FlowFiles in the order in which they were transferred
- * to the given relationship
+ * to the given relationship
*/
public List<MockFlowFile> getFlowFilesForRelationship(final String relationship) {
final Relationship procRel = new Relationship.Builder().name(relationship).build();
@@ -778,7 +778,7 @@ public class MockProcessSession implements ProcessSession {
*/
private FlowFile inheritAttributes(final FlowFile source, final FlowFile destination) {
if (source == null || destination == null || source == destination) {
- return destination; //don't need to inherit from ourselves
+ return destination; // don't need to inherit from ourselves
}
final FlowFile updated = putAllAttributes(destination, source.getAttributes());
getProvenanceReporter().fork(source, Collections.singletonList(updated));
@@ -801,7 +801,7 @@ public class MockProcessSession implements ProcessSession {
int uuidsCaptured = 0;
for (final FlowFile source : sources) {
if (source == destination) {
- continue; //don't want to capture parent uuid of this. Something can't be a child of itself
+ continue; // don't want to capture parent uuid of this. Something can't be a child of itself
}
final String sourceUuid = source.getAttribute(CoreAttributes.UUID.key());
if (sourceUuid != null && !sourceUuid.trim().isEmpty()) {
@@ -832,7 +832,7 @@ public class MockProcessSession implements ProcessSession {
*/
private static Map<String, String> intersectAttributes(final Collection<FlowFile> flowFileList) {
final Map<String, String> result = new HashMap<>();
- //trivial cases
+ // trivial cases
if (flowFileList == null || flowFileList.isEmpty()) {
return result;
} else if (flowFileList.size() == 1) {
@@ -845,8 +845,7 @@ public class MockProcessSession implements ProcessSession {
*/
final Map<String, String> firstMap = flowFileList.iterator().next().getAttributes();
- outer:
- for (final Map.Entry<String, String> mapEntry : firstMap.entrySet()) {
+ outer: for (final Map.Entry<String, String> mapEntry : firstMap.entrySet()) {
final String key = mapEntry.getKey();
final String value = mapEntry.getValue();
for (final FlowFile flowFile : flowFileList) {
@@ -900,7 +899,7 @@ public class MockProcessSession implements ProcessSession {
public void assertTransferCount(final Relationship relationship, final int count) {
final int transferCount = getFlowFilesForRelationship(relationship).size();
Assert.assertEquals("Expected " + count + " FlowFiles to be transferred to "
- + relationship + " but actual transfer count was " + transferCount, count, transferCount);
+ + relationship + " but actual transfer count was " + transferCount, count, transferCount);
}
/**
http://git-wip-us.apache.org/repos/asf/nifi/blob/49a781df/nifi-mock/src/main/java/org/apache/nifi/util/StandardProcessorTestRunner.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/StandardProcessorTestRunner.java b/nifi-mock/src/main/java/org/apache/nifi/util/StandardProcessorTestRunner.java
index eeeff61..0d00cc8 100644
--- a/nifi-mock/src/main/java/org/apache/nifi/util/StandardProcessorTestRunner.java
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/StandardProcessorTestRunner.java
@@ -58,12 +58,12 @@ import org.apache.nifi.components.ValidationContext;
import org.apache.nifi.components.ValidationResult;
import org.apache.nifi.controller.ConfigurationContext;
import org.apache.nifi.controller.ControllerService;
+import org.apache.nifi.controller.queue.QueueSize;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.flowfile.attributes.CoreAttributes;
import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.processor.ProcessSessionFactory;
import org.apache.nifi.processor.Processor;
-import org.apache.nifi.processor.QueueSize;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.provenance.ProvenanceEventRecord;
import org.apache.nifi.provenance.ProvenanceReporter;
@@ -222,7 +222,7 @@ public class StandardProcessorTestRunner implements TestRunner {
boolean unscheduledRun = false;
for (final Future<Throwable> future : futures) {
try {
- final Throwable thrown = future.get(); // wait for the result
+ final Throwable thrown = future.get(); // wait for the result
if (thrown != null) {
throw new AssertionError(thrown);
}
@@ -551,11 +551,11 @@ public class StandardProcessorTestRunner implements TestRunner {
@Override
public void addControllerService(final String identifier, final ControllerService service, final Map<String, String> properties) throws InitializationException {
// hold off on failing due to deprecated annotation for now... will introduce later.
-// for ( final Method method : service.getClass().getMethods() ) {
-// if ( method.isAnnotationPresent(org.apache.nifi.controller.annotation.OnConfigured.class) ) {
-// Assert.fail("Controller Service " + service + " is using deprecated Annotation " + org.apache.nifi.controller.annotation.OnConfigured.class + " for method " + method);
-// }
-// }
+ // for ( final Method method : service.getClass().getMethods() ) {
+ // if ( method.isAnnotationPresent(org.apache.nifi.controller.annotation.OnConfigured.class) ) {
+ // Assert.fail("Controller Service " + service + " is using deprecated Annotation " + org.apache.nifi.controller.annotation.OnConfigured.class + " for method " + method);
+ // }
+ // }
final ComponentLog logger = new MockProcessorLog(identifier, service);
final MockControllerServiceInitializationContext initContext = new MockControllerServiceInitializationContext(requireNonNull(service), requireNonNull(identifier), logger);
@@ -716,11 +716,11 @@ public class StandardProcessorTestRunner implements TestRunner {
final PropertyDescriptor descriptor = service.getPropertyDescriptor(propertyName);
if (descriptor == null) {
return new ValidationResult.Builder()
- .input(propertyName)
- .explanation(propertyName + " is not a known Property for Controller Service " + service)
- .subject("Invalid property")
- .valid(false)
- .build();
+ .input(propertyName)
+ .explanation(propertyName + " is not a known Property for Controller Service " + service)
+ .subject("Invalid property")
+ .valid(false)
+ .build();
}
return setProperty(service, descriptor, value);
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/49a781df/nifi-mock/src/main/java/org/apache/nifi/util/TestRunner.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/TestRunner.java b/nifi-mock/src/main/java/org/apache/nifi/util/TestRunner.java
index 6e66bfe..ec901fe 100644
--- a/nifi-mock/src/main/java/org/apache/nifi/util/TestRunner.java
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/TestRunner.java
@@ -26,11 +26,11 @@ import org.apache.nifi.components.AllowableValue;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.ValidationResult;
import org.apache.nifi.controller.ControllerService;
+import org.apache.nifi.controller.queue.QueueSize;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.ProcessSessionFactory;
import org.apache.nifi.processor.Processor;
-import org.apache.nifi.processor.QueueSize;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.provenance.ProvenanceEventRecord;
import org.apache.nifi.provenance.ProvenanceReporter;
@@ -40,22 +40,22 @@ public interface TestRunner {
/**
* @return the {@link Processor} for which this <code>TestRunner</code> is
- * configured
+ * configured
*/
Processor getProcessor();
/**
* @return the {@link ProcessSessionFactory} that this
- * <code>TestRunner</code> will use to invoke the
- * {@link Processor#onTrigger(ProcessContext, ProcessSessionFactory)} method
+ * <code>TestRunner</code> will use to invoke the
+ * {@link Processor#onTrigger(ProcessContext, ProcessSessionFactory)} method
*/
ProcessSessionFactory getProcessSessionFactory();
/**
* @return the {@Link ProcessContext} that this <code>TestRunner</code> will
- * use to invoke the
- * {@link Processor#onTrigger(ProcessContext, ProcessSessionFactory) onTrigger}
- * method
+ * use to invoke the
+ * {@link Processor#onTrigger(ProcessContext, ProcessSessionFactory) onTrigger}
+ * method
*/
ProcessContext getProcessContext();
@@ -120,7 +120,7 @@ public interface TestRunner {
*
* @param iterations number of iterations
* @param stopOnFinish whether or not to run the Processor methods that are
- * annotated with {@link org.apache.nifi.processor.annotation.OnStopped @OnStopped}
+ * annotated with {@link org.apache.nifi.processor.annotation.OnStopped @OnStopped}
* @param initialize true if must initialize
*/
void run(int iterations, boolean stopOnFinish, final boolean initialize);
@@ -163,10 +163,10 @@ public interface TestRunner {
*
* @param iterations number of iterations
* @param stopOnFinish whether or not to run the Processor methods that are
- * annotated with {@link org.apache.nifi.processor.annotation.OnStopped @OnStopped}
+ * annotated with {@link org.apache.nifi.processor.annotation.OnStopped @OnStopped}
* @param initialize true if must initialize
* @param runWait indicates the amount of time in milliseconds that the framework should wait for
- * processors to stop running before calling the {@link org.apache.nifi.processor.annotation.OnUnscheduled @OnUnscheduled} annotation
+ * processors to stop running before calling the {@link org.apache.nifi.processor.annotation.OnUnscheduled @OnUnscheduled} annotation
*/
void run(int iterations, boolean stopOnFinish, final boolean initialize, final long runWait);
@@ -187,8 +187,8 @@ public interface TestRunner {
/**
* @return the currently configured number of threads that will be used to
- * runt he Processor when calling the {@link #run()} or {@link #run(int)}
- * methods
+ * runt he Processor when calling the {@link #run()} or {@link #run(int)}
+ * methods
*/
int getThreadCount();
@@ -296,7 +296,7 @@ public interface TestRunner {
/**
* @return <code>true</code> if the Input Queue to the Processor is empty,
- * <code>false</code> otherwise
+ * <code>false</code> otherwise
*/
boolean isQueueEmpty();
@@ -421,7 +421,7 @@ public interface TestRunner {
/**
* @return the {@link ProvenanceReporter} that will be used by the
- * configured {@link Processor} for reporting Provenance Events
+ * configured {@link Processor} for reporting Provenance Events
*/
ProvenanceReporter getProvenanceReporter();
@@ -433,7 +433,7 @@ public interface TestRunner {
/**
* @param name of counter
* @return the current value of the counter with the specified name, or null
- * if no counter exists with the specified name
+ * if no counter exists with the specified name
*/
Long getCounterValue(String name);
@@ -599,14 +599,14 @@ public interface TestRunner {
/**
* @param service the service
* @return {@code true} if the given Controller Service is enabled,
- * {@code false} if it is disabled
+ * {@code false} if it is disabled
*
* @throws IllegalArgumentException if the given ControllerService is not
- * known by this TestRunner (i.e., it has not been added via the
- * {@link #addControllerService(String, ControllerService)} or
- * {@link #addControllerService(String, ControllerService, Map)} method or
- * if the Controller Service has been removed via the
- * {@link #removeControllerService(ControllerService)} method.
+ * known by this TestRunner (i.e., it has not been added via the
+ * {@link #addControllerService(String, ControllerService)} or
+ * {@link #addControllerService(String, ControllerService, Map)} method or
+ * if the Controller Service has been removed via the
+ * {@link #removeControllerService(ControllerService)} method.
*/
boolean isControllerServiceEnabled(ControllerService service);
@@ -622,11 +622,11 @@ public interface TestRunner {
*
* @throws IllegalStateException if the ControllerService is not disabled
* @throws IllegalArgumentException if the given ControllerService is not
- * known by this TestRunner (i.e., it has not been added via the
- * {@link #addControllerService(String, ControllerService)} or
- * {@link #addControllerService(String, ControllerService, Map)} method or
- * if the Controller Service has been removed via the
- * {@link #removeControllerService(ControllerService)} method.
+ * known by this TestRunner (i.e., it has not been added via the
+ * {@link #addControllerService(String, ControllerService)} or
+ * {@link #addControllerService(String, ControllerService, Map)} method or
+ * if the Controller Service has been removed via the
+ * {@link #removeControllerService(ControllerService)} method.
*
*/
void removeControllerService(ControllerService service);
@@ -641,11 +641,11 @@ public interface TestRunner {
*
* @throws IllegalStateException if the ControllerService is not disabled
* @throws IllegalArgumentException if the given ControllerService is not
- * known by this TestRunner (i.e., it has not been added via the
- * {@link #addControllerService(String, ControllerService)} or
- * {@link #addControllerService(String, ControllerService, Map)} method or
- * if the Controller Service has been removed via the
- * {@link #removeControllerService(ControllerService)} method.
+ * known by this TestRunner (i.e., it has not been added via the
+ * {@link #addControllerService(String, ControllerService)} or
+ * {@link #addControllerService(String, ControllerService, Map)} method or
+ * if the Controller Service has been removed via the
+ * {@link #removeControllerService(ControllerService)} method.
*
*/
ValidationResult setProperty(ControllerService service, PropertyDescriptor property, String value);
@@ -660,11 +660,11 @@ public interface TestRunner {
*
* @throws IllegalStateException if the ControllerService is not disabled
* @throws IllegalArgumentException if the given ControllerService is not
- * known by this TestRunner (i.e., it has not been added via the
- * {@link #addControllerService(String, ControllerService)} or
- * {@link #addControllerService(String, ControllerService, Map)} method or
- * if the Controller Service has been removed via the
- * {@link #removeControllerService(ControllerService)} method.
+ * known by this TestRunner (i.e., it has not been added via the
+ * {@link #addControllerService(String, ControllerService)} or
+ * {@link #addControllerService(String, ControllerService, Map)} method or
+ * if the Controller Service has been removed via the
+ * {@link #removeControllerService(ControllerService)} method.
*
*/
ValidationResult setProperty(ControllerService service, PropertyDescriptor property, AllowableValue value);
@@ -679,11 +679,11 @@ public interface TestRunner {
*
* @throws IllegalStateException if the ControllerService is not disabled
* @throws IllegalArgumentException if the given ControllerService is not
- * known by this TestRunner (i.e., it has not been added via the
- * {@link #addControllerService(String, ControllerService)} or
- * {@link #addControllerService(String, ControllerService, Map)} method or
- * if the Controller Service has been removed via the
- * {@link #removeControllerService(ControllerService)} method.
+ * known by this TestRunner (i.e., it has not been added via the
+ * {@link #addControllerService(String, ControllerService)} or
+ * {@link #addControllerService(String, ControllerService, Map)} method or
+ * if the Controller Service has been removed via the
+ * {@link #removeControllerService(ControllerService)} method.
*
*/
ValidationResult setProperty(ControllerService service, String propertyName, String value);
@@ -698,19 +698,19 @@ public interface TestRunner {
* @throws IllegalStateException if the Controller Service is not disabled
*
* @throws IllegalArgumentException if the given ControllerService is not
- * known by this TestRunner (i.e., it has not been added via the
- * {@link #addControllerService(String, ControllerService)} or
- * {@link #addControllerService(String, ControllerService, Map)} method or
- * if the Controller Service has been removed via the
- * {@link #removeControllerService(ControllerService)} method.
+ * known by this TestRunner (i.e., it has not been added via the
+ * {@link #addControllerService(String, ControllerService)} or
+ * {@link #addControllerService(String, ControllerService, Map)} method or
+ * if the Controller Service has been removed via the
+ * {@link #removeControllerService(ControllerService)} method.
*/
void setAnnotationData(ControllerService service, String annotationData);
/**
* @param identifier of controller service
* @return the {@link ControllerService} that is registered with the given
- * identifier, or <code>null</code> if no Controller Service exists with the
- * given identifier
+ * identifier, or <code>null</code> if no Controller Service exists with the
+ * given identifier
*/
ControllerService getControllerService(String identifier);
@@ -720,11 +720,11 @@ public interface TestRunner {
*
* @param service the service to validate
* @throws IllegalArgumentException if the given ControllerService is not
- * known by this TestRunner (i.e., it has not been added via the
- * {@link #addControllerService(String, ControllerService)} or
- * {@link #addControllerService(String, ControllerService, Map)} method or
- * if the Controller Service has been removed via the
- * {@link #removeControllerService(ControllerService)} method.
+ * known by this TestRunner (i.e., it has not been added via the
+ * {@link #addControllerService(String, ControllerService)} or
+ * {@link #addControllerService(String, ControllerService, Map)} method or
+ * if the Controller Service has been removed via the
+ * {@link #removeControllerService(ControllerService)} method.
*/
void assertValid(ControllerService service);
@@ -734,11 +734,11 @@ public interface TestRunner {
*
* @param service the service to validate
* @throws IllegalArgumentException if the given ControllerService is not
- * known by this TestRunner (i.e., it has not been added via the
- * {@link #addControllerService(String, ControllerService)} or
- * {@link #addControllerService(String, ControllerService, Map)} method or
- * if the Controller Service has been removed via the
- * {@link #removeControllerService(ControllerService)} method.
+ * known by this TestRunner (i.e., it has not been added via the
+ * {@link #addControllerService(String, ControllerService)} or
+ * {@link #addControllerService(String, ControllerService, Map)} method or
+ * if the Controller Service has been removed via the
+ * {@link #removeControllerService(ControllerService)} method.
*
*/
void assertNotValid(ControllerService service);
@@ -748,12 +748,12 @@ public interface TestRunner {
* @param identifier identifier of service
* @param serviceType type of service
* @return the {@link ControllerService} that is registered with the given
- * identifier, cast as the provided service type, or <code>null</code> if no
- * Controller Service exists with the given identifier
+ * identifier, cast as the provided service type, or <code>null</code> if no
+ * Controller Service exists with the given identifier
*
* @throws ClassCastException if the identifier given is registered for a
- * Controller Service but that Controller Service is not of the type
- * specified
+ * Controller Service but that Controller Service is not of the type
+ * specified
*/
<T extends ControllerService> T getControllerService(String identifier, Class<T> serviceType);
http://git-wip-us.apache.org/repos/asf/nifi/blob/49a781df/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/DropFlowFileRequest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/DropFlowFileRequest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/DropFlowFileRequest.java
new file mode 100644
index 0000000..609fe75
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/DropFlowFileRequest.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.controller;
+
+import org.apache.nifi.controller.queue.DropFlowFileState;
+import org.apache.nifi.controller.queue.DropFlowFileStatus;
+import org.apache.nifi.controller.queue.QueueSize;
+
+public class DropFlowFileRequest implements DropFlowFileStatus {
+ private final String identifier;
+ private final long submissionTime = System.currentTimeMillis();
+
+ private volatile QueueSize originalSize;
+ private volatile QueueSize currentSize;
+ private volatile long lastUpdated = System.currentTimeMillis();
+ private volatile Thread executionThread;
+
+ private DropFlowFileState state = DropFlowFileState.WAITING_FOR_LOCK;
+
+
+ public DropFlowFileRequest(final String identifier) {
+ this.identifier = identifier;
+ }
+
+ @Override
+ public String getRequestIdentifier() {
+ return identifier;
+ }
+
+ @Override
+ public long getRequestSubmissionTime() {
+ return submissionTime;
+ }
+
+ @Override
+ public QueueSize getOriginalSize() {
+ return originalSize;
+ }
+
+ void setOriginalSize(final QueueSize originalSize) {
+ this.originalSize = originalSize;
+ }
+
+ @Override
+ public QueueSize getCurrentSize() {
+ return currentSize;
+ }
+
+ void setCurrentSize(final QueueSize queueSize) {
+ this.currentSize = currentSize;
+ }
+
+ @Override
+ public DropFlowFileState getState() {
+ return state;
+ }
+
+ @Override
+ public long getLastUpdated() {
+ return lastUpdated;
+ }
+
+ synchronized void setState(final DropFlowFileState state) {
+ this.state = state;
+ this.lastUpdated = System.currentTimeMillis();
+ }
+
+ void setExecutionThread(final Thread thread) {
+ this.executionThread = thread;
+ }
+
+ synchronized boolean cancel() {
+ if (this.state == DropFlowFileState.COMPLETE || this.state == DropFlowFileState.CANCELED) {
+ return false;
+ }
+
+ this.state = DropFlowFileState.CANCELED;
+ if (executionThread != null) {
+ executionThread.interrupt();
+ }
+
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/49a781df/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
index df356fd..073e5fb 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
@@ -16,6 +16,7 @@
*/
package org.apache.nifi.controller;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
@@ -24,9 +25,13 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import java.util.PriorityQueue;
import java.util.Queue;
import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
@@ -35,25 +40,32 @@ import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.nifi.connectable.Connection;
+import org.apache.nifi.controller.queue.DropFlowFileState;
import org.apache.nifi.controller.queue.DropFlowFileStatus;
import org.apache.nifi.controller.queue.FlowFileQueue;
import org.apache.nifi.controller.queue.QueueSize;
import org.apache.nifi.controller.repository.FlowFileRecord;
+import org.apache.nifi.controller.repository.FlowFileRepository;
import org.apache.nifi.controller.repository.FlowFileSwapManager;
+import org.apache.nifi.controller.repository.RepositoryRecord;
+import org.apache.nifi.controller.repository.RepositoryRecordType;
import org.apache.nifi.controller.repository.claim.ContentClaim;
+import org.apache.nifi.controller.repository.claim.ResourceClaim;
+import org.apache.nifi.controller.repository.claim.ResourceClaimManager;
import org.apache.nifi.events.EventReporter;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.flowfile.FlowFilePrioritizer;
import org.apache.nifi.processor.DataUnit;
import org.apache.nifi.processor.FlowFileFilter;
import org.apache.nifi.processor.FlowFileFilter.FlowFileFilterResult;
+import org.apache.nifi.provenance.ProvenanceEventBuilder;
+import org.apache.nifi.provenance.ProvenanceEventRecord;
+import org.apache.nifi.provenance.ProvenanceEventRepository;
+import org.apache.nifi.provenance.ProvenanceEventType;
import org.apache.nifi.reporting.Severity;
import org.apache.nifi.scheduling.SchedulingStrategy;
import org.apache.nifi.util.FormatUtils;
import org.apache.nifi.util.concurrency.TimedLock;
-import org.apache.nifi.util.timebuffer.LongEntityAccess;
-import org.apache.nifi.util.timebuffer.TimedBuffer;
-import org.apache.nifi.util.timebuffer.TimestampedLong;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -67,15 +79,6 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
public static final int MAX_EXPIRED_RECORDS_PER_ITERATION = 100000;
public static final int SWAP_RECORD_POLL_SIZE = 10000;
- // When we have very high contention on a FlowFile Queue, the writeLock quickly becomes the bottleneck. In order to avoid this,
- // we keep track of how often we are obtaining the write lock. If we exceed some threshold, we start performing a Pre-fetch so that
- // we can then poll many times without having to obtain the lock.
- // If lock obtained an average of more than PREFETCH_POLL_THRESHOLD times per second in order to poll from queue for last 5 seconds, do a pre-fetch.
- public static final int PREFETCH_POLL_THRESHOLD = 1000;
- public static final int PRIORITIZED_PREFETCH_SIZE = 10;
- public static final int UNPRIORITIZED_PREFETCH_SIZE = 1000;
- private volatile int prefetchSize = UNPRIORITIZED_PREFETCH_SIZE; // when we pre-fetch, how many should we pre-fetch?
-
private static final Logger logger = LoggerFactory.getLogger(StandardFlowFileQueue.class);
private PriorityQueue<FlowFileRecord> activeQueue = null;
@@ -97,9 +100,13 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
private final List<FlowFilePrioritizer> priorities;
private final int swapThreshold;
private final FlowFileSwapManager swapManager;
+ private final List<String> swapLocations = new ArrayList<>();
private final TimedLock readLock;
private final TimedLock writeLock;
private final String identifier;
+ private final FlowFileRepository flowFileRepository;
+ private final ProvenanceEventRepository provRepository;
+ private final ResourceClaimManager resourceClaimManager;
private final AtomicBoolean queueFullRef = new AtomicBoolean(false);
private final AtomicInteger activeQueueSizeRef = new AtomicInteger(0);
@@ -108,8 +115,8 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
// SCHEDULER CANNOT BE NOTIFIED OF EVENTS WITH THE WRITE LOCK HELD! DOING SO WILL RESULT IN A DEADLOCK!
private final ProcessScheduler scheduler;
- public StandardFlowFileQueue(final String identifier, final Connection connection, final ProcessScheduler scheduler, final FlowFileSwapManager swapManager, final EventReporter eventReporter,
- final int swapThreshold) {
+ public StandardFlowFileQueue(final String identifier, final Connection connection, final FlowFileRepository flowFileRepo, final ProvenanceEventRepository provRepo,
+ final ResourceClaimManager resourceClaimManager, final ProcessScheduler scheduler, final FlowFileSwapManager swapManager, final EventReporter eventReporter, final int swapThreshold) {
activeQueue = new PriorityQueue<>(20, new Prioritizer(new ArrayList<FlowFilePrioritizer>()));
priorities = new ArrayList<>();
maximumQueueObjectCount = 0L;
@@ -120,6 +127,9 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
swapQueue = new ArrayList<>();
this.eventReporter = eventReporter;
this.swapManager = swapManager;
+ this.flowFileRepository = flowFileRepo;
+ this.provRepository = provRepo;
+ this.resourceClaimManager = resourceClaimManager;
this.identifier = identifier;
this.swapThreshold = swapThreshold;
@@ -141,11 +151,6 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
}
@Override
- public int getSwapThreshold() {
- return swapThreshold;
- }
-
- @Override
public void setPriorities(final List<FlowFilePrioritizer> newPriorities) {
writeLock.lock();
try {
@@ -154,12 +159,6 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
activeQueue = newQueue;
priorities.clear();
priorities.addAll(newPriorities);
-
- if (newPriorities.isEmpty()) {
- prefetchSize = UNPRIORITIZED_PREFETCH_SIZE;
- } else {
- prefetchSize = PRIORITIZED_PREFETCH_SIZE;
- }
} finally {
writeLock.unlock("setPriorities");
}
@@ -225,33 +224,16 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
*/
private QueueSize getQueueSize() {
final QueueSize unacknowledged = unacknowledgedSizeRef.get();
- final PreFetch preFetch = preFetchRef.get();
- final int preFetchCount;
- final long preFetchSize;
- if (preFetch == null) {
- preFetchCount = 0;
- preFetchSize = 0L;
- } else {
- final QueueSize preFetchQueueSize = preFetch.size();
- preFetchCount = preFetchQueueSize.getObjectCount();
- preFetchSize = preFetchQueueSize.getByteCount();
- }
-
- return new QueueSize(activeQueue.size() + swappedRecordCount + unacknowledged.getObjectCount() + preFetchCount,
- activeQueueContentSize + swappedContentSize + unacknowledged.getByteCount() + preFetchSize);
+ return new QueueSize(activeQueue.size() + swappedRecordCount + unacknowledged.getObjectCount(),
+ activeQueueContentSize + swappedContentSize + unacknowledged.getByteCount());
}
@Override
public boolean isEmpty() {
readLock.lock();
try {
- final PreFetch prefetch = preFetchRef.get();
- if (prefetch == null) {
- return activeQueue.isEmpty() && swappedRecordCount == 0 && unacknowledgedSizeRef.get().getObjectCount() == 0;
- } else {
- return activeQueue.isEmpty() && swappedRecordCount == 0 && unacknowledgedSizeRef.get().getObjectCount() == 0 && prefetch.size().getObjectCount() == 0;
- }
+ return activeQueue.isEmpty() && swappedRecordCount == 0 && unacknowledgedSizeRef.get().getObjectCount() == 0;
} finally {
readLock.unlock("isEmpty");
}
@@ -260,30 +242,13 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
@Override
public boolean isActiveQueueEmpty() {
final int activeQueueSize = activeQueueSizeRef.get();
- if (activeQueueSize == 0) {
- final PreFetch preFetch = preFetchRef.get();
- if (preFetch == null) {
- return true;
- }
-
- final QueueSize queueSize = preFetch.size();
- return queueSize.getObjectCount() == 0;
- } else {
- return false;
- }
+ return activeQueueSize == 0;
}
- @Override
public QueueSize getActiveQueueSize() {
readLock.lock();
try {
- final PreFetch preFetch = preFetchRef.get();
- if (preFetch == null) {
- return new QueueSize(activeQueue.size(), activeQueueContentSize);
- } else {
- final QueueSize preFetchSize = preFetch.size();
- return new QueueSize(activeQueue.size() + preFetchSize.getObjectCount(), activeQueueContentSize + preFetchSize.getByteCount());
- }
+ return new QueueSize(activeQueue.size(), activeQueueContentSize);
} finally {
readLock.unlock("getActiveQueueSize");
}
@@ -374,6 +339,7 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
swappedContentSize += file.getSize();
swappedRecordCount++;
swapMode = true;
+ writeSwapFilesIfNecessary();
} else {
activeQueueContentSize += file.getSize();
activeQueue.add(file);
@@ -405,6 +371,7 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
swappedContentSize += bytes;
swappedRecordCount += numFiles;
swapMode = true;
+ writeSwapFilesIfNecessary();
} else {
activeQueueContentSize += bytes;
activeQueue.addAll(files);
@@ -421,116 +388,6 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
}
}
- @Override
- public List<FlowFileRecord> pollSwappableRecords() {
- writeLock.lock();
- try {
- if (swapQueue.size() < SWAP_RECORD_POLL_SIZE) {
- return null;
- }
-
- final List<FlowFileRecord> swapRecords = new ArrayList<>(Math.min(SWAP_RECORD_POLL_SIZE, swapQueue.size()));
- final Iterator<FlowFileRecord> itr = swapQueue.iterator();
- while (itr.hasNext() && swapRecords.size() < SWAP_RECORD_POLL_SIZE) {
- final FlowFileRecord record = itr.next();
- swapRecords.add(record);
- itr.remove();
- }
-
- swapQueue.trimToSize();
- return swapRecords;
- } finally {
- writeLock.unlock("pollSwappableRecords");
- }
- }
-
- @Override
- public void putSwappedRecords(final Collection<FlowFileRecord> records) {
- writeLock.lock();
- try {
- try {
- for (final FlowFileRecord record : records) {
- swappedContentSize -= record.getSize();
- swappedRecordCount--;
- activeQueueContentSize += record.getSize();
- activeQueue.add(record);
- }
-
- if (swappedRecordCount > swapQueue.size()) {
- // we have more swap files to be swapped in.
- return;
- }
-
- // If a call to #pollSwappableRecords will not produce any, go ahead and roll those FlowFiles back into the mix
- if (swapQueue.size() < SWAP_RECORD_POLL_SIZE) {
- for (final FlowFileRecord record : swapQueue) {
- activeQueue.add(record);
- activeQueueContentSize += record.getSize();
- }
- swapQueue.clear();
- swappedContentSize = 0L;
- swappedRecordCount = 0;
- swapMode = false;
- }
- } finally {
- activeQueueSizeRef.set(activeQueue.size());
- }
- } finally {
- writeLock.unlock("putSwappedRecords");
- scheduler.registerEvent(connection.getDestination());
- }
- }
-
- @Override
- public void incrementSwapCount(final int numRecords, final long contentSize) {
- writeLock.lock();
- try {
- swappedContentSize += contentSize;
- swappedRecordCount += numRecords;
- } finally {
- writeLock.unlock("incrementSwapCount");
- }
- }
-
- @Override
- public int unswappedSize() {
- readLock.lock();
- try {
- return activeQueue.size() + unacknowledgedSizeRef.get().getObjectCount();
- } finally {
- readLock.unlock("unswappedSize");
- }
- }
-
- @Override
- public int getSwapRecordCount() {
- readLock.lock();
- try {
- return swappedRecordCount;
- } finally {
- readLock.unlock("getSwapRecordCount");
- }
- }
-
- @Override
- public int getSwapQueueSize() {
- readLock.lock();
- try {
- if (logger.isDebugEnabled()) {
- final long byteToMbDivisor = 1024L * 1024L;
- final QueueSize unacknowledged = unacknowledgedSizeRef.get();
-
- logger.debug("Total Queue Size: ActiveQueue={}/{} MB, Swap Queue={}/{} MB, Unacknowledged={}/{} MB",
- activeQueue.size(), activeQueueContentSize / byteToMbDivisor,
- swappedRecordCount, swappedContentSize / byteToMbDivisor,
- unacknowledged.getObjectCount(), unacknowledged.getByteCount() / byteToMbDivisor);
- }
-
- return swapQueue.size();
- } finally {
- readLock.unlock("getSwapQueueSize");
- }
- }
private boolean isLaterThan(final Long maxAge) {
if (maxAge == null) {
@@ -558,30 +415,6 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
// First check if we have any records Pre-Fetched.
final long expirationMillis = flowFileExpirationMillis.get();
- final PreFetch preFetch = preFetchRef.get();
- if (preFetch != null) {
- if (preFetch.isExpired()) {
- requeueExpiredPrefetch(preFetch);
- } else {
- while (true) {
- final FlowFileRecord next = preFetch.nextRecord();
- if (next == null) {
- break;
- }
-
- if (isLaterThan(getExpirationDate(next, expirationMillis))) {
- expiredRecords.add(next);
- continue;
- }
-
- updateUnacknowledgedSize(1, next.getSize());
- return next;
- }
-
- preFetchRef.compareAndSet(preFetch, null);
- }
- }
-
writeLock.lock();
try {
flowFile = doPoll(expiredRecords, expirationMillis);
@@ -631,9 +464,6 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
queueFullRef.set(determineIfFull());
}
- if (incrementPollCount()) {
- prefetch();
- }
return isExpired ? null : flowFile;
}
@@ -642,38 +472,6 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
final List<FlowFileRecord> records = new ArrayList<>(Math.min(1024, maxResults));
// First check if we have any records Pre-Fetched.
- final long expirationMillis = flowFileExpirationMillis.get();
- final PreFetch preFetch = preFetchRef.get();
- if (preFetch != null) {
- if (preFetch.isExpired()) {
- requeueExpiredPrefetch(preFetch);
- } else {
- long totalSize = 0L;
- for (int i = 0; i < maxResults; i++) {
- final FlowFileRecord next = preFetch.nextRecord();
- if (next == null) {
- break;
- }
-
- if (isLaterThan(getExpirationDate(next, expirationMillis))) {
- expiredRecords.add(next);
- continue;
- }
-
- records.add(next);
- totalSize += next.getSize();
- }
-
- // If anything was prefetched, use what we have.
- if (!records.isEmpty()) {
- updateUnacknowledgedSize(records.size(), totalSize);
- return records;
- }
-
- preFetchRef.compareAndSet(preFetch, null);
- }
- }
-
writeLock.lock();
try {
doPoll(records, maxResults, expiredRecords);
@@ -705,10 +503,6 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
if (queueFullAtStart && !expiredRecords.isEmpty()) {
queueFullRef.set(determineIfFull());
}
-
- if (incrementPollCount()) {
- prefetch();
- }
}
/**
@@ -732,6 +526,46 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
// Swap Queue to the Active Queue. However, we don't do this if there are FlowFiles already swapped out
// to disk, because we want them to be swapped back in in the same order that they were swapped out.
+ if (activeQueue.size() > swapThreshold - SWAP_RECORD_POLL_SIZE) {
+ return;
+ }
+
+ // If there are swap files waiting to be swapped in, swap those in first. We do this in order to ensure that those that
+ // were swapped out first are then swapped back in first. If we instead just immediately migrated the FlowFiles from the
+ // swap queue to the active queue, and we never run out of FlowFiles in the active queue (because destination cannot
+ // keep up with queue), we will end up always processing the new FlowFiles first instead of the FlowFiles that arrived
+ // first.
+ if (!swapLocations.isEmpty()) {
+ final String swapLocation = swapLocations.remove(0);
+ try {
+ final List<FlowFileRecord> swappedIn = swapManager.swapIn(swapLocation, this);
+ swappedRecordCount -= swappedIn.size();
+ long swapSize = 0L;
+ for (final FlowFileRecord flowFile : swappedIn) {
+ swapSize += flowFile.getSize();
+ }
+ swappedContentSize -= swapSize;
+ activeQueueContentSize += swapSize;
+ activeQueueSizeRef.set(activeQueue.size());
+ activeQueue.addAll(swappedIn);
+ return;
+ } catch (final FileNotFoundException fnfe) {
+ logger.error("Failed to swap in FlowFiles from Swap File {} because the Swap File can no longer be found", swapLocation);
+ if (eventReporter != null) {
+ eventReporter.reportEvent(Severity.ERROR, "Swap File", "Failed to swap in FlowFiles from Swap File " + swapLocation + " because the Swap File can no longer be found");
+ }
+ return;
+ } catch (final IOException ioe) {
+ logger.error("Failed to swap in FlowFiles from Swap File {}; Swap File appears to be corrupt!", swapLocation);
+ logger.error("", ioe);
+ if (eventReporter != null) {
+ eventReporter.reportEvent(Severity.ERROR, "Swap File", "Failed to swap in FlowFiles from Swap File " +
+ swapLocation + "; Swap File appears to be corrupt! Some FlowFiles in the queue may not be accessible. See logs for more information.");
+ }
+ return;
+ }
+ }
+
// this is the most common condition (nothing is swapped out), so do the check first and avoid the expense
// of other checks for 99.999% of the cases.
if (swappedRecordCount == 0 && swapQueue.isEmpty()) {
@@ -760,6 +594,69 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
}
}
+ /**
+ * This method MUST be called with the write lock held
+ */
+ private void writeSwapFilesIfNecessary() {
+ if (swapQueue.size() < SWAP_RECORD_POLL_SIZE) {
+ return;
+ }
+
+ final int numSwapFiles = swapQueue.size() / SWAP_RECORD_POLL_SIZE;
+
+ // Create a new Priority queue with the prioritizers that are set, but reverse the
+ // prioritizers because we want to pull the lowest-priority FlowFiles to swap out
+ final PriorityQueue<FlowFileRecord> tempQueue = new PriorityQueue<>(activeQueue.size() + swapQueue.size(), Collections.reverseOrder(new Prioritizer(priorities)));
+ tempQueue.addAll(activeQueue);
+ tempQueue.addAll(swapQueue);
+
+ final List<String> swapLocations = new ArrayList<>(numSwapFiles);
+ for (int i = 0; i < numSwapFiles; i++) {
+ // Create a new swap file for the next SWAP_RECORD_POLL_SIZE records
+ final List<FlowFileRecord> toSwap = new ArrayList<>(SWAP_RECORD_POLL_SIZE);
+ for (int j = 0; j < SWAP_RECORD_POLL_SIZE; j++) {
+ toSwap.add(tempQueue.poll());
+ }
+
+ try {
+ Collections.reverse(toSwap); // currently ordered in reverse priority order based on the ordering of the temp queue.
+ final String swapLocation = swapManager.swapOut(toSwap, this);
+ swapLocations.add(swapLocation);
+ } catch (final IOException ioe) {
+ tempQueue.addAll(toSwap); // if we failed, we must add the FlowFiles back to the queue.
+ logger.error("FlowFile Queue with identifier {} has {} FlowFiles queued up. Attempted to spill FlowFile information over to disk in order to avoid exhausting "
+ + "the Java heap space but failed to write information to disk due to {}", getIdentifier(), getQueueSize().getObjectCount(), ioe.toString());
+ logger.error("", ioe);
+ if (eventReporter != null) {
+ eventReporter.reportEvent(Severity.ERROR, "Failed to Overflow to Disk", "Flowfile Queue with identifier " + getIdentifier() + " has " + getQueueSize().getObjectCount() +
+ " queued up. Attempted to spill FlowFile information over to disk in order to avoid exhausting the Java heap space but failed to write information to disk. "
+ + "See logs for more information.");
+ }
+
+ break;
+ }
+ }
+
+ // Pull any records off of the temp queue that won't fit back on the active queue, and add those to the
+ // swap queue. Then add the records back to the active queue.
+ swapQueue.clear();
+ while (tempQueue.size() > swapThreshold) {
+ final FlowFileRecord record = tempQueue.poll();
+ swapQueue.add(record);
+ }
+
+ Collections.reverse(swapQueue); // currently ordered in reverse priority order based on the ordering of the temp queue
+
+ // replace the contents of the active queue, since we've merged it with the swap queue.
+ activeQueue.clear();
+ FlowFileRecord toRequeue;
+ while ((toRequeue = tempQueue.poll()) != null) {
+ activeQueue.offer(toRequeue);
+ }
+ this.swapLocations.addAll(swapLocations);
+ }
+
+
@Override
public long drainQueue(final Queue<FlowFileRecord> sourceQueue, final List<FlowFileRecord> destination, int maxResults, final Set<FlowFileRecord> expiredRecords) {
long drainedSize = 0L;
@@ -796,13 +693,6 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
final List<FlowFileRecord> selectedFlowFiles = new ArrayList<>();
final List<FlowFileRecord> unselected = new ArrayList<>();
- // the prefetch doesn't allow us to add records back. So when this method is used,
- // if there are prefetched records, we have to requeue them into the active queue first.
- final PreFetch prefetch = preFetchRef.get();
- if (prefetch != null) {
- requeueExpiredPrefetch(prefetch);
- }
-
while (true) {
FlowFileRecord flowFile = this.activeQueue.poll();
if (flowFile == null) {
@@ -856,6 +746,8 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
}
}
+
+
private static final class Prioritizer implements Comparator<FlowFileRecord>, Serializable {
private static final long serialVersionUID = 1L;
@@ -991,6 +883,7 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
this.swappedRecordCount = swapFlowFileCount;
this.swappedContentSize = swapByteCount;
+ this.swapLocations.addAll(swapLocations);
} finally {
writeLock.unlock("Recover Swap Files");
}
@@ -1004,173 +897,219 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
return "FlowFileQueue[id=" + identifier + "]";
}
- @Override
- public DropFlowFileStatus dropFlowFiles() {
- // TODO Auto-generated method stub
- return null;
- }
+ private final ConcurrentMap<String, DropFlowFileRequest> dropRequestMap = new ConcurrentHashMap<>();
@Override
- public boolean cancelDropFlowFileRequest(String requestIdentifier) {
- // TODO Auto-generated method stub
- return false;
- }
+ public DropFlowFileStatus dropFlowFiles() {
+ // purge any old requests from the map just to keep it clean. But if there are very requests, which is usually the case, then don't bother
+ if (dropRequestMap.size() > 10) {
+ final List<String> toDrop = new ArrayList<>();
+ for (final Map.Entry<String, DropFlowFileRequest> entry : dropRequestMap.entrySet()) {
+ final DropFlowFileRequest request = entry.getValue();
+ final boolean completed = request.getState() == DropFlowFileState.COMPLETE || request.getState() == DropFlowFileState.FAILURE;
+
+ if (completed && System.currentTimeMillis() - request.getLastUpdated() > TimeUnit.MINUTES.toMillis(5L)) {
+ toDrop.add(entry.getKey());
+ }
+ }
- @Override
- public DropFlowFileStatus getDropFlowFileStatus(String requestIdentifier) {
- // TODO Auto-generated method stub
- return null;
- }
+ for (final String requestId : toDrop) {
+ dropRequestMap.remove(requestId);
+ }
+ }
- /**
- * Lock the queue so that other threads are unable to interact with the
- * queue
- */
- public void lock() {
- writeLock.lock();
- }
+ // TODO: get user name!
+ final String userName = null;
- /**
- * Unlock the queue
- */
- public void unlock() {
- writeLock.unlock("external unlock");
- }
+ final String requestIdentifier = UUID.randomUUID().toString();
+ final DropFlowFileRequest dropRequest = new DropFlowFileRequest(requestIdentifier);
+ final Thread t = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ writeLock.lock();
+ try {
+ dropRequest.setState(DropFlowFileState.DROPPING_ACTIVE_FLOWFILES);
+
+ try {
+ final List<FlowFileRecord> activeQueueRecords = new ArrayList<>(activeQueue);
+ drop(activeQueueRecords, userName);
+ activeQueue.clear();
+ dropRequest.setCurrentSize(getQueueSize());
+
+ drop(swapQueue, userName);
+ swapQueue.clear();
+ dropRequest.setCurrentSize(getQueueSize());
+
+ final Iterator<String> swapLocationItr = swapLocations.iterator();
+ while (swapLocationItr.hasNext()) {
+ final String swapLocation = swapLocationItr.next();
+ final List<FlowFileRecord> swappedIn = swapManager.swapIn(swapLocation, StandardFlowFileQueue.this);
+ try {
+ drop(swappedIn, userName);
+ } catch (final Exception e) {
+ activeQueue.addAll(swappedIn); // ensure that we don't lose the FlowFiles from our queue.
+ throw e;
+ }
+
+ dropRequest.setCurrentSize(getQueueSize());
+ swapLocationItr.remove();
+ }
- @Override
- public QueueSize getUnacknowledgedQueueSize() {
- return unacknowledgedSizeRef.get();
- }
+ dropRequest.setState(DropFlowFileState.COMPLETE);
+ } catch (final Exception e) {
+ // TODO: Handle adequately
+ dropRequest.setState(DropFlowFileState.FAILURE);
+ }
+ } finally {
+ writeLock.unlock("Drop FlowFiles");
+ }
+ }
+ }, "Drop FlowFiles for Connection " + getIdentifier());
+ t.setDaemon(true);
+ t.start();
- private void updateUnacknowledgedSize(final int addToCount, final long addToSize) {
- boolean updated = false;
+ dropRequest.setExecutionThread(t);
+ dropRequestMap.put(requestIdentifier, dropRequest);
- do {
- final QueueSize queueSize = unacknowledgedSizeRef.get();
- final QueueSize newSize = new QueueSize(queueSize.getObjectCount() + addToCount, queueSize.getByteCount() + addToSize);
- updated = unacknowledgedSizeRef.compareAndSet(queueSize, newSize);
- } while (!updated);
+ return dropRequest;
}
- private void requeueExpiredPrefetch(final PreFetch prefetch) {
- if (prefetch == null) {
- return;
+ private void drop(final List<FlowFileRecord> flowFiles, final String user) throws IOException {
+ // Create a Provenance Event and a FlowFile Repository record for each FlowFile
+ final List<ProvenanceEventRecord> provenanceEvents = new ArrayList<>(flowFiles.size());
+ final List<RepositoryRecord> flowFileRepoRecords = new ArrayList<>(flowFiles.size());
+ for (final FlowFileRecord flowFile : flowFiles) {
+ provenanceEvents.add(createDropEvent(flowFile, user));
+ flowFileRepoRecords.add(createDeleteRepositoryRecord(flowFile));
}
- writeLock.lock();
- try {
- final long contentSizeRequeued = prefetch.requeue(activeQueue);
- this.activeQueueContentSize += contentSizeRequeued;
- this.preFetchRef.compareAndSet(prefetch, null);
- } finally {
- writeLock.unlock("requeueExpiredPrefetch");
- }
- }
+ for (final FlowFileRecord flowFile : flowFiles) {
+ final ContentClaim contentClaim = flowFile.getContentClaim();
+ if (contentClaim == null) {
+ continue;
+ }
- /**
- * MUST be called with write lock held.
- */
- private final AtomicReference<PreFetch> preFetchRef = new AtomicReference<>();
+ final ResourceClaim resourceClaim = contentClaim.getResourceClaim();
+ if (resourceClaim == null) {
+ continue;
+ }
- private void prefetch() {
- if (activeQueue.isEmpty()) {
- return;
+ resourceClaimManager.decrementClaimantCount(resourceClaim);
}
- final int numToFetch = Math.min(prefetchSize, activeQueue.size());
+ provRepository.registerEvents(provenanceEvents);
+ flowFileRepository.updateRepository(flowFileRepoRecords);
+ }
- final PreFetch curPreFetch = preFetchRef.get();
- if (curPreFetch != null && curPreFetch.size().getObjectCount() > 0) {
- return;
- }
+ private ProvenanceEventRecord createDropEvent(final FlowFileRecord flowFile, final String user) {
+ final ProvenanceEventBuilder builder = provRepository.eventBuilder();
+ builder.fromFlowFile(flowFile);
+ builder.setEventType(ProvenanceEventType.DROP);
+ builder.setLineageStartDate(flowFile.getLineageStartDate());
+ builder.setComponentId(getIdentifier());
+ builder.setComponentType("Connection");
+ builder.setDetails("FlowFile manually dropped by user " + user);
+ return builder.build();
+ }
- final List<FlowFileRecord> buffer = new ArrayList<>(numToFetch);
- long contentSize = 0L;
- for (int i = 0; i < numToFetch; i++) {
- final FlowFileRecord record = activeQueue.poll();
- if (record == null || record.isPenalized()) {
- // not enough unpenalized records to pull. Put all records back and return
- activeQueue.addAll(buffer);
- if (record != null) {
- activeQueue.add(record);
- }
- return;
- } else {
- buffer.add(record);
- contentSize += record.getSize();
+ private RepositoryRecord createDeleteRepositoryRecord(final FlowFileRecord flowFile) {
+ return new RepositoryRecord() {
+ @Override
+ public FlowFileQueue getDestination() {
+ return null;
}
- }
- activeQueueContentSize -= contentSize;
- preFetchRef.set(new PreFetch(buffer));
- }
+ @Override
+ public FlowFileQueue getOriginalQueue() {
+ return StandardFlowFileQueue.this;
+ }
- private final TimedBuffer<TimestampedLong> pollCounts = new TimedBuffer<>(TimeUnit.SECONDS, 5, new LongEntityAccess());
+ @Override
+ public RepositoryRecordType getType() {
+ return RepositoryRecordType.DELETE;
+ }
- private boolean incrementPollCount() {
- pollCounts.add(new TimestampedLong(1L));
- final long totalCount = pollCounts.getAggregateValue(System.currentTimeMillis() - 5000L).getValue();
- return totalCount > PREFETCH_POLL_THRESHOLD * 5;
- }
+ @Override
+ public ContentClaim getCurrentClaim() {
+ return flowFile.getContentClaim();
+ }
- private static class PreFetch {
+ @Override
+ public ContentClaim getOriginalClaim() {
+ return flowFile.getContentClaim();
+ }
- private final List<FlowFileRecord> records;
- private final AtomicInteger pointer = new AtomicInteger(0);
- private final long expirationTime = System.nanoTime() + TimeUnit.SECONDS.toNanos(1L);
- private final AtomicLong contentSize = new AtomicLong(0L);
+ @Override
+ public long getCurrentClaimOffset() {
+ return flowFile.getContentClaimOffset();
+ }
- public PreFetch(final List<FlowFileRecord> records) {
- this.records = records;
+ @Override
+ public FlowFileRecord getCurrent() {
+ return flowFile;
+ }
- long totalSize = 0L;
- for (final FlowFileRecord record : records) {
- totalSize += record.getSize();
+ @Override
+ public boolean isAttributesChanged() {
+ return false;
}
- contentSize.set(totalSize);
- }
- public FlowFileRecord nextRecord() {
- final int nextValue = pointer.getAndIncrement();
- if (nextValue >= records.size()) {
+ @Override
+ public boolean isMarkedForAbort() {
+ return false;
+ }
+
+ @Override
+ public String getSwapLocation() {
return null;
}
+ };
+ }
- final FlowFileRecord flowFile = records.get(nextValue);
- contentSize.addAndGet(-flowFile.getSize());
- return flowFile;
+
+ @Override
+ public boolean cancelDropFlowFileRequest(final String requestIdentifier) {
+ final DropFlowFileRequest request = dropRequestMap.remove(requestIdentifier);
+ if (request == null) {
+ return false;
}
- public QueueSize size() {
- final int pointerIndex = pointer.get();
- final int count = records.size() - pointerIndex;
- if (count < 0) {
- return new QueueSize(0, 0L);
- }
+ final boolean successful = request.cancel();
+ return successful;
+ }
- final long bytes = contentSize.get();
- return new QueueSize(count, bytes);
- }
+ @Override
+ public DropFlowFileStatus getDropFlowFileStatus(final String requestIdentifier) {
+ return dropRequestMap.get(requestIdentifier);
+ }
- public boolean isExpired() {
- return System.nanoTime() > expirationTime;
- }
+ /**
+ * Lock the queue so that other threads are unable to interact with the
+ * queue
+ */
+ public void lock() {
+ writeLock.lock();
+ }
- private long requeue(final Queue<FlowFileRecord> queue) {
- // get the current pointer and prevent any other thread from accessing the rest of the elements
- final int curPointer = pointer.getAndAdd(records.size());
- if (curPointer < records.size() - 1) {
- final List<FlowFileRecord> subList = records.subList(curPointer, records.size());
- long contentSize = 0L;
- for (final FlowFileRecord record : subList) {
- contentSize += record.getSize();
- }
+ /**
+ * Unlock the queue
+ */
+ public void unlock() {
+ writeLock.unlock("external unlock");
+ }
- queue.addAll(subList);
+ @Override
+ public QueueSize getUnacknowledgedQueueSize() {
+ return unacknowledgedSizeRef.get();
+ }
- return contentSize;
- }
- return 0L;
- }
+ private void updateUnacknowledgedSize(final int addToCount, final long addToSize) {
+ boolean updated = false;
+
+ do {
+ final QueueSize queueSize = unacknowledgedSizeRef.get();
+ final QueueSize newSize = new QueueSize(queueSize.getObjectCount() + addToCount, queueSize.getByteCount() + addToSize);
+ updated = unacknowledgedSizeRef.compareAndSet(queueSize, newSize);
+ } while (!updated);
}
}
[03/30] nifi git commit: NIFI-730: - Starting to add support for
deleting flow files from a queue by creating endpoints and starting to wire
everything together. - Adding context menu item for initiating the request to
drop flow files.
Posted by ma...@apache.org.
NIFI-730:
- Starting to add support for deleting flow files from a queue by creating endpoints and starting to wire everything together.
- Adding context menu item for initiating the request to drop flow files.
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/e0ac7cde
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/e0ac7cde
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/e0ac7cde
Branch: refs/heads/master
Commit: e0ac7cde372f428b0655465b7adc59ad41f8f270
Parents: b4bfcc1
Author: Matt Gilman <ma...@gmail.com>
Authored: Mon Oct 12 10:00:54 2015 -0400
Committer: Matt Gilman <ma...@gmail.com>
Committed: Mon Oct 12 10:00:54 2015 -0400
----------------------------------------------------------------------
.../apache/nifi/web/api/dto/DropRequestDTO.java | 129 +++++++++++
.../nifi/web/api/entity/DropRequestEntity.java | 44 ++++
.../org/apache/nifi/web/NiFiServiceFacade.java | 25 +++
.../nifi/web/StandardNiFiServiceFacade.java | 24 ++
.../apache/nifi/web/api/ConnectionResource.java | 217 ++++++++++++++++++-
.../org/apache/nifi/web/dao/ConnectionDAO.java | 22 ++
.../web/dao/impl/StandardConnectionDAO.java | 15 ++
.../src/main/webapp/js/nf/canvas/nf-actions.js | 58 +++++
.../main/webapp/js/nf/canvas/nf-context-menu.js | 10 +
9 files changed, 542 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/e0ac7cde/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/DropRequestDTO.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/DropRequestDTO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/DropRequestDTO.java
new file mode 100644
index 0000000..dd4289f
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/DropRequestDTO.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.api.dto;
+
+import com.wordnik.swagger.annotations.ApiModelProperty;
+import java.util.Date;
+import javax.xml.bind.annotation.XmlType;
+import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
+import org.apache.nifi.web.api.dto.util.TimestampAdapter;
+
+/**
+ * A request to drop the contents of a connection.
+ */
+@XmlType(name = "dropRequest")
+public class DropRequestDTO {
+
+ private String id;
+ private String uri;
+
+ private Date submissionTime;
+ private Date expiration;
+
+ private Integer percentCompleted;
+ private Boolean finished;
+
+ /**
+ * The id for this component.
+ *
+ * @return The id
+ */
+ @ApiModelProperty(
+ value = "The id of the component."
+ )
+ public String getId() {
+ return this.id;
+ }
+
+ public void setId(final String id) {
+ this.id = id;
+ }
+
+ /**
+ * The uri for linking to this component in this NiFi.
+ *
+ * @return The uri
+ */
+ @ApiModelProperty(
+ value = "The URI for futures requests to the component."
+ )
+ public String getUri() {
+ return uri;
+ }
+
+ public void setUri(String uri) {
+ this.uri = uri;
+ }
+
+ /**
+ * @return time the query was submitted
+ */
+ @XmlJavaTypeAdapter(TimestampAdapter.class)
+ @ApiModelProperty(
+ value = "The timestamp when the query was submitted."
+ )
+ public Date getSubmissionTime() {
+ return submissionTime;
+ }
+
+ public void setSubmissionTime(Date submissionTime) {
+ this.submissionTime = submissionTime;
+ }
+
+ /**
+ * @return expiration time of the query results
+ */
+ @XmlJavaTypeAdapter(TimestampAdapter.class)
+ @ApiModelProperty(
+ value = "The timestamp when the query will expire."
+ )
+ public Date getExpiration() {
+ return expiration;
+ }
+
+ public void setExpiration(Date expiration) {
+ this.expiration = expiration;
+ }
+
+ /**
+ * @return percent completed
+ */
+ @ApiModelProperty(
+ value = "The current percent complete."
+ )
+ public Integer getPercentCompleted() {
+ return percentCompleted;
+ }
+
+ public void setPercentCompleted(Integer percentCompleted) {
+ this.percentCompleted = percentCompleted;
+ }
+
+ /**
+ * @return whether the query has finished
+ */
+ @ApiModelProperty(
+ value = "Whether the query has finished."
+ )
+ public Boolean isFinished() {
+ return finished;
+ }
+
+ public void setFinished(Boolean finished) {
+ this.finished = finished;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/e0ac7cde/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/DropRequestEntity.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/DropRequestEntity.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/DropRequestEntity.java
new file mode 100644
index 0000000..078c019
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/DropRequestEntity.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.api.entity;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.apache.nifi.web.api.dto.DropRequestDTO;
+
+/**
+ * A serialized representation of this class can be placed in the entity body of a request or response to or from the API. This particular entity holds a reference to a DropRequestDTO.
+ */
+@XmlRootElement(name = "dropRequestEntity")
+public class DropRequestEntity extends Entity {
+
+ private DropRequestDTO dropRequest;
+
+ /**
+ * The DropRequestDTO that is being serialized.
+ *
+ * @return The DropRequestDTO object
+ */
+ public DropRequestDTO getDropRequest() {
+ return dropRequest;
+ }
+
+ public void setDropRequest(DropRequestDTO dropRequest) {
+ this.dropRequest = dropRequest;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/e0ac7cde/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java
index c98b1e4..28f6b61 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java
@@ -43,6 +43,7 @@ import org.apache.nifi.web.api.dto.ProcessGroupDTO;
import org.apache.nifi.web.api.dto.ProcessorDTO;
import org.apache.nifi.web.api.dto.ComponentHistoryDTO;
import org.apache.nifi.web.api.dto.ControllerServiceReferencingComponentDTO;
+import org.apache.nifi.web.api.dto.DropRequestDTO;
import org.apache.nifi.web.api.dto.PropertyDescriptorDTO;
import org.apache.nifi.web.api.dto.RemoteProcessGroupDTO;
import org.apache.nifi.web.api.dto.RemoteProcessGroupPortDTO;
@@ -525,6 +526,30 @@ public interface NiFiServiceFacade {
*/
ConfigurationSnapshot<Void> deleteConnection(Revision revision, String groupId, String connectionId);
+ /**
+ * Creates a new flow file drop request.
+ *
+ * @param groupId group
+ * @param connectionId The ID of the connection
+ * @return
+ */
+ DropRequestDTO createFlowFileDropRequest(String groupId, String connectionId);
+
+ /**
+ * Gets the specified flow file drop request.
+ *
+ * @param dropRequestId The flow file drop request
+ * @return The DropRequest
+ */
+ DropRequestDTO getFlowFileDropRequest(String dropRequestId);
+
+ /**
+ * Cancels/removes the specified flow file drop request.
+ *
+ * @param dropRequestId The flow file drop request
+ */
+ void deleteFlowFileDropRequest(String dropRequestId);
+
// ----------------------------------------
// InputPort methods
// ----------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/e0ac7cde/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java
index 2286213..7f0a296 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java
@@ -162,6 +162,7 @@ import org.apache.nifi.controller.service.ControllerServiceState;
import org.apache.nifi.reporting.ComponentType;
import org.apache.nifi.web.api.dto.ControllerServiceDTO;
import org.apache.nifi.web.api.dto.ControllerServiceReferencingComponentDTO;
+import org.apache.nifi.web.api.dto.DropRequestDTO;
import org.apache.nifi.web.api.dto.PropertyDescriptorDTO;
import org.apache.nifi.web.api.dto.ReportingTaskDTO;
import org.apache.nifi.web.api.dto.status.ClusterProcessGroupStatusDTO;
@@ -809,6 +810,11 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade {
}
@Override
+ public void deleteFlowFileDropRequest(String dropRequestId) {
+ // TODO
+ }
+
+ @Override
public ConfigurationSnapshot<Void> deleteProcessor(final Revision revision, final String groupId, final String processorId) {
return optimisticLockingManager.configureFlow(revision, new ConfigurationRequest<Void>() {
@Override
@@ -1060,6 +1066,18 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade {
}
@Override
+ public DropRequestDTO createFlowFileDropRequest(String groupId, String connectionId) {
+ // TODO
+ final DropRequestDTO dto = new DropRequestDTO();
+ dto.setFinished(false);
+ dto.setSubmissionTime(new Date());
+ dto.setExpiration(new Date(System.currentTimeMillis() + 10000));
+ dto.setId(UUID.randomUUID().toString());
+ dto.setPercentCompleted(100);
+ return dto;
+ }
+
+ @Override
public ConfigurationSnapshot<ProcessorDTO> createProcessor(final Revision revision, final String groupId, final ProcessorDTO processorDTO) {
return optimisticLockingManager.configureFlow(revision, new ConfigurationRequest<ProcessorDTO>() {
@Override
@@ -2092,6 +2110,12 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade {
}
@Override
+ public DropRequestDTO getFlowFileDropRequest(String dropRequestId) {
+ // TODO
+ return null;
+ }
+
+ @Override
public StatusHistoryDTO getConnectionStatusHistory(String groupId, String connectionId) {
return controllerFacade.getConnectionStatusHistory(groupId, connectionId);
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/e0ac7cde/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectionResource.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectionResource.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectionResource.java
index 64c14fa..dfc20fb 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectionResource.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectionResource.java
@@ -24,6 +24,7 @@ import com.wordnik.swagger.annotations.ApiResponses;
import com.wordnik.swagger.annotations.Authorization;
import java.net.URI;
import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -49,6 +50,7 @@ import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.core.Response;
+import javax.ws.rs.core.Response.Status;
import org.apache.nifi.cluster.manager.impl.WebClusterManager;
import org.apache.nifi.util.NiFiProperties;
@@ -71,6 +73,10 @@ import org.apache.nifi.web.api.request.IntegerParameter;
import org.apache.nifi.web.api.request.LongParameter;
import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.cluster.context.ClusterContext;
+import org.apache.nifi.cluster.context.ClusterContextThreadLocal;
+import org.apache.nifi.web.api.dto.DropRequestDTO;
+import org.apache.nifi.web.api.entity.DropRequestEntity;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.security.access.prepost.PreAuthorize;
@@ -476,8 +482,7 @@ public class ConnectionResource extends ApplicationResource {
@ApiParam(
value = "The connection configuration details.",
required = true
- )
- ConnectionEntity connectionEntity) {
+ ) ConnectionEntity connectionEntity) {
if (connectionEntity == null || connectionEntity.getConnection() == null) {
throw new IllegalArgumentException("Connection details must be specified.");
@@ -886,6 +891,214 @@ public class ConnectionResource extends ApplicationResource {
return clusterContext(generateOkResponse(entity)).build();
}
+ /**
+ * Drops the flowfiles in the queue of the specified connection.
+ *
+ * @param clientId Optional client id. If the client id is not specified, a new one will be generated. This value (whether specified or generated) is included in the response.
+ * @param id The id of the connection
+ * @return A dropRequestEntity
+ */
+ @DELETE
+ @Consumes(MediaType.WILDCARD)
+ @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML})
+ @Path("/{connection-id}/contents")
+ @PreAuthorize("hasRole('ROLE_DFM')")
+ @ApiOperation(
+ value = "Drops the contents of the queue in this connection.",
+ response = DropRequestEntity.class,
+ authorizations = {
+ @Authorization(value = "Data Flow Manager", type = "ROLE_DFM")
+ }
+ )
+ @ApiResponses(
+ value = {
+ @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."),
+ @ApiResponse(code = 401, message = "Client could not be authenticated."),
+ @ApiResponse(code = 403, message = "Client is not authorized to make this request."),
+ @ApiResponse(code = 404, message = "The specified resource could not be found."),
+ @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful.")
+ }
+ )
+ public Response dropQueueContents(
+ @ApiParam(
+ value = "If the client id is not specified, new one will be generated. This value (whether specified or generated) is included in the response.",
+ required = false
+ )
+ @QueryParam(CLIENT_ID) @DefaultValue(StringUtils.EMPTY) ClientIdParameter clientId,
+ @ApiParam(
+ value = "The connection id.",
+ required = true
+ )
+ @PathParam("connection-id") String id) {
+
+ // replicate if cluster manager
+ if (properties.isClusterManager()) {
+ return clusterManager.applyRequest(HttpMethod.DELETE, getAbsolutePath(), getRequestParameters(true), getHeaders()).getResponse();
+ }
+
+ // ensure the id is the same across the cluster
+ final String dropRequestId;
+ final ClusterContext clusterContext = ClusterContextThreadLocal.getContext();
+ if (clusterContext != null) {
+ dropRequestId = UUID.nameUUIDFromBytes(clusterContext.getIdGenerationSeed().getBytes(StandardCharsets.UTF_8)).toString();
+ } else {
+ dropRequestId = UUID.randomUUID().toString();
+ }
+
+ // submit the drop request
+ final DropRequestDTO dropRequest = serviceFacade.createFlowFileDropRequest(groupId, id);
+ dropRequest.setUri(generateResourceUri("controller", "process-groups", groupId, "connections", id, "contents", "drop-requests", dropRequestId));
+
+ // create the revision
+ final RevisionDTO revision = new RevisionDTO();
+ revision.setClientId(clientId.getClientId());
+
+ // create the response entity
+ final DropRequestEntity entity = new DropRequestEntity();
+ entity.setRevision(revision);
+ entity.setDropRequest(dropRequest);
+
+ // generate the URI where the response will be
+ final URI location = URI.create(dropRequest.getUri());
+ if (dropRequest.isFinished()) {
+ return generateCreatedResponse(location, entity).build();
+ } else {
+ return Response.status(Status.ACCEPTED).location(location).entity(entity).build();
+ }
+ }
+
+ /**
+ * Checks the status of an outstanding drop request.
+ *
+ * @param clientId Optional client id. If the client id is not specified, a new one will be generated. This value (whether specified or generated) is included in the response.
+ * @param connectionId The id of the connection
+ * @param dropRequestId The id of the drop request
+ * @return A dropRequestEntity
+ */
+ @GET
+ @Consumes(MediaType.WILDCARD)
+ @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML})
+ @Path("/{connection-id}/contents/drop-requests/{drop-request-id}")
+ @PreAuthorize("hasRole('ROLE_DFM')")
+ @ApiOperation(
+ value = "Gets the current status of a drop request for the specified connection.",
+ response = DropRequestEntity.class,
+ authorizations = {
+ @Authorization(value = "Data Flow Manager", type = "ROLE_DFM")
+ }
+ )
+ @ApiResponses(
+ value = {
+ @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."),
+ @ApiResponse(code = 401, message = "Client could not be authenticated."),
+ @ApiResponse(code = 403, message = "Client is not authorized to make this request."),
+ @ApiResponse(code = 404, message = "The specified resource could not be found."),
+ @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful.")
+ }
+ )
+ public Response getDropRequest(
+ @ApiParam(
+ value = "If the client id is not specified, new one will be generated. This value (whether specified or generated) is included in the response.",
+ required = false
+ )
+ @QueryParam(CLIENT_ID) @DefaultValue(StringUtils.EMPTY) ClientIdParameter clientId,
+ @ApiParam(
+ value = "The connection id.",
+ required = true
+ )
+ @PathParam("connection-id") String connectionId,
+ @ApiParam(
+ value = "The drop request id.",
+ required = true
+ )
+ @PathParam("drop-request-id") String dropRequestId) {
+
+ // replicate if cluster manager
+ if (properties.isClusterManager()) {
+ return clusterManager.applyRequest(HttpMethod.GET, getAbsolutePath(), getRequestParameters(true), getHeaders()).getResponse();
+ }
+
+ // get the drop request
+ final DropRequestDTO dropRequest = serviceFacade.getFlowFileDropRequest(dropRequestId);
+ dropRequest.setUri(generateResourceUri("controller", "process-groups", groupId, "connections", connectionId, "contents", "drop-requests", dropRequestId));
+
+ // create the revision
+ final RevisionDTO revision = new RevisionDTO();
+ revision.setClientId(clientId.getClientId());
+
+ // create the response entity
+ final DropRequestEntity entity = new DropRequestEntity();
+ entity.setRevision(revision);
+ entity.setDropRequest(dropRequest);
+
+ return generateOkResponse(entity).build();
+ }
+
+ /**
+ * Deletes the specified drop request.
+ *
+ * @param clientId Optional client id. If the client id is not specified, a new one will be generated. This value (whether specified or generated) is included in the response.
+ * @param connectionId The connection id
+ * @param dropRequestId The drop request id
+ * @return A dropRequestEntity
+ */
+ @DELETE
+ @Consumes(MediaType.WILDCARD)
+ @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML})
+ @Path("/{connection-id}/contents/drop-requests/{drop-request-id}")
+ @PreAuthorize("hasRole('ROLE_DFM')")
+ @ApiOperation(
+ value = "Cancels and/or removes a request drop of the contents in this connection.",
+ response = DropRequestEntity.class,
+ authorizations = {
+ @Authorization(value = "Data Flow Manager", type = "ROLE_DFM")
+ }
+ )
+ @ApiResponses(
+ value = {
+ @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."),
+ @ApiResponse(code = 401, message = "Client could not be authenticated."),
+ @ApiResponse(code = 403, message = "Client is not authorized to make this request."),
+ @ApiResponse(code = 404, message = "The specified resource could not be found."),
+ @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful.")
+ }
+ )
+ public Response removeDropRequest(
+ @ApiParam(
+ value = "If the client id is not specified, new one will be generated. This value (whether specified or generated) is included in the response.",
+ required = false
+ )
+ @QueryParam(CLIENT_ID) @DefaultValue(StringUtils.EMPTY) ClientIdParameter clientId,
+ @ApiParam(
+ value = "The connection id.",
+ required = true
+ )
+ @PathParam("connection-id") String connectionId,
+ @ApiParam(
+ value = "The drop request id.",
+ required = true
+ )
+ @PathParam("drop-request-id") String dropRequestId) {
+
+ // replicate if cluster manager
+ if (properties.isClusterManager()) {
+ return clusterManager.applyRequest(HttpMethod.DELETE, getAbsolutePath(), getRequestParameters(true), getHeaders()).getResponse();
+ }
+
+ // delete the drop request
+ serviceFacade.deleteFlowFileDropRequest(dropRequestId);
+
+ // create the revision
+ final RevisionDTO revision = new RevisionDTO();
+ revision.setClientId(clientId.getClientId());
+
+ // create the response entity
+ final DropRequestEntity entity = new DropRequestEntity();
+ entity.setRevision(revision);
+
+ return generateOkResponse(entity).build();
+ }
+
// setters
public void setServiceFacade(NiFiServiceFacade serviceFacade) {
this.serviceFacade = serviceFacade;
http://git-wip-us.apache.org/repos/asf/nifi/blob/e0ac7cde/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectionDAO.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectionDAO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectionDAO.java
index e0fb89e..ce1d1fd 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectionDAO.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectionDAO.java
@@ -32,6 +32,13 @@ public interface ConnectionDAO {
Connection getConnection(String groupId, String id);
/**
+ * Gets the specified flow file drop request.
+ *
+ * @param dropRequestId The drop request id
+ */
+ void getFlowFileDropRequest(String dropRequestId);
+
+ /**
* Gets the connections for the specified source processor.
*
* @param groupId group id
@@ -67,6 +74,14 @@ public interface ConnectionDAO {
Connection createConnection(String groupId, ConnectionDTO connectionDTO);
/**
+ * Creates a new flow file drop request.
+ *
+ * @param groupId group id
+ * @param id connection id
+ */
+ void createFileFlowDropRequest(String groupId, String id);
+
+ /**
* Verifies the create request can be processed.
*
* @param groupId group id
@@ -106,4 +121,11 @@ public interface ConnectionDAO {
* @param id The id of the connection
*/
void deleteConnection(String groupId, String id);
+
+ /**
+ * Deletes the specified flow file drop request.
+ *
+ * @param dropRequestId The drop request id
+ */
+ void deleteFlowFileDropRequest(String dropRequestId);
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/e0ac7cde/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java
index 5fbc393..8fa9d3b 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java
@@ -69,6 +69,11 @@ public class StandardConnectionDAO extends ComponentDAO implements ConnectionDAO
}
@Override
+ public void getFlowFileDropRequest(String dropRequestId) {
+ // TODO
+ }
+
+ @Override
public Set<Connection> getConnectionsForSource(final String groupId, final String processorId) {
final Set<Connection> connections = new HashSet<>(getConnections(groupId));
for (final Iterator<Connection> connectionIter = connections.iterator(); connectionIter.hasNext();) {
@@ -294,6 +299,11 @@ public class StandardConnectionDAO extends ComponentDAO implements ConnectionDAO
}
@Override
+ public void createFileFlowDropRequest(String groupId, String id) {
+ // TODO
+ }
+
+ @Override
public void verifyCreate(String groupId, ConnectionDTO connectionDTO) {
// validate the incoming request
final List<String> validationErrors = validateProposedConfiguration(groupId, connectionDTO);
@@ -464,6 +474,11 @@ public class StandardConnectionDAO extends ComponentDAO implements ConnectionDAO
group.removeConnection(connection);
}
+ @Override
+ public void deleteFlowFileDropRequest(String dropRequestId) {
+ // TODO
+ }
+
/* setters */
public void setFlowController(final FlowController flowController) {
this.flowController = flowController;
http://git-wip-us.apache.org/repos/asf/nifi/blob/e0ac7cde/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js
index 3b47a8d..bab2236 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js
@@ -846,6 +846,64 @@ nf.Actions = (function () {
},
/**
+ * Deletes the flow files in the specified connection.
+ *
+ * @param {type} selection
+ */
+ deleteQueueContents: function (selection) {
+ if (selection.size() !== 1 || !nf.CanvasUtils.isConnection(selection)) {
+ return;
+ }
+
+ // process the drop request
+ var processDropRequest = function (dropRequest, nextDelay) {
+ // see if the drop request has completed
+ if (dropRequest.finished === true) {
+ deleteDropRequest(dropRequest);
+ } else {
+ schedule(dropRequest, nextDelay);
+ }
+ };
+
+ // schedule for the next poll iteration
+ var schedule = function (dropRequest, delay) {
+ setTimeout(function () {
+ $.ajax({
+ type: 'GET',
+ url: dropRequest.uri,
+ dataType: 'json'
+ }).done(function(response) {
+ var dropRequest = response.dropRequest;
+ processDropRequest(dropRequest, Math.min(8, delay * 2));
+ }).fail(nf.Common.handleAjaxError);
+ }, delay * 1000);
+ };
+
+ // delete the drop request
+ var deleteDropRequest = function (dropRequest) {
+ $.ajax({
+ type: 'DELETE',
+ url: dropRequest.uri,
+ dataType: 'json'
+ }).done(function() {
+ // drop request has been deleted
+ }).fail(nf.Common.handleAjaxError);
+ };
+
+ // get the connection data
+ var connection = selection.datum();
+
+ // issue the request to delete the flow files
+ $.ajax({
+ type: 'DELETE',
+ url: connection.component.uri + '/contents',
+ dataType: 'json'
+ }).done(function(response) {
+ processDropRequest(response.dropRequest, 1);
+ }).fail(nf.Common.handleAjaxError);
+ },
+
+ /**
* Opens the fill color dialog for the component in the specified selection.
*
* @param {type} selection The selection
http://git-wip-us.apache.org/repos/asf/nifi/blob/e0ac7cde/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-context-menu.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-context-menu.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-context-menu.js
index e652dd4..58397d4 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-context-menu.js
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-context-menu.js
@@ -278,6 +278,15 @@ nf.ContextMenu = (function () {
};
/**
+ * Only DFMs can delete flow files from a connection.
+ *
+ * @param {selection} selection
+ */
+ var canDeleteFlowFiles = function (selection) {
+ return nf.Common.isDFM() && isConnection(selection);
+ };
+
+ /**
* Determines if the components in the specified selection can be moved into a parent group.
*
* @param {type} selection
@@ -373,6 +382,7 @@ nf.ContextMenu = (function () {
{condition: isCopyable, menuItem: {img: 'images/iconCopy.png', text: 'Copy', action: 'copy'}},
{condition: isPastable, menuItem: {img: 'images/iconPaste.png', text: 'Paste', action: 'paste'}},
{condition: canMoveToParent, menuItem: {img: 'images/iconMoveToParent.png', text: 'Move to parent group', action: 'moveIntoParent'}},
+ {condition: canDeleteFlowFiles, menuItem: {img: 'images/iconDelete.png', text: 'Delete Flow Files', action: 'deleteQueueContents'}},
{condition: isDeletable, menuItem: {img: 'images/iconDelete.png', text: 'Delete', action: 'delete'}}
];
[02/30] nifi git commit: NIFI-730: Added methods for dropping queued
flowfiles;
refactored swap manager but have not yet started swapping flowfiles in or out
from within the flowfile queue
Posted by ma...@apache.org.
NIFI-730: Added methods for dropping queued flowfiles; refactored swap manager but have not yet started swapping flowfiles in or out from within the flowfile queue
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/b8c51dc3
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/b8c51dc3
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/b8c51dc3
Branch: refs/heads/master
Commit: b8c51dc35d1a7fdbf3e6449bbe297db667a1176c
Parents: b4bfcc1
Author: Mark Payne <ma...@hotmail.com>
Authored: Sun Oct 11 10:27:07 2015 -0400
Committer: Mark Payne <ma...@hotmail.com>
Committed: Sun Oct 11 10:27:07 2015 -0400
----------------------------------------------------------------------
.../apache/nifi/controller/FlowFileQueue.java | 203 ------
.../controller/queue/DropFlowFileState.java | 40 ++
.../controller/queue/DropFlowFileStatus.java | 62 ++
.../nifi/controller/queue/FlowFileQueue.java | 256 ++++++++
.../apache/nifi/controller/queue/QueueSize.java | 48 ++
.../repository/FlowFileRepository.java | 2 +-
.../repository/FlowFileSwapManager.java | 94 ++-
.../controller/repository/QueueProvider.java | 2 +-
.../controller/repository/RepositoryRecord.java | 2 +-
.../SwapManagerInitializationContext.java | 41 ++
.../apache/nifi/processor/ProcessSession.java | 1 +
.../org/apache/nifi/processor/QueueSize.java | 48 --
.../org/apache/nifi/connectable/Connection.java | 2 +-
.../nifi/controller/StandardFlowFileQueue.java | 108 +++-
.../nifi/connectable/StandardConnection.java | 26 +-
.../nifi/controller/FileSystemSwapManager.java | 626 ++++++++-----------
.../apache/nifi/controller/FlowController.java | 193 +++---
.../repository/BatchingSessionFactory.java | 2 +-
.../repository/ConnectionSwapInfo.java | 58 --
.../repository/StandardProcessSession.java | 4 +-
.../repository/StandardRepositoryRecord.java | 2 +-
.../repository/VolatileFlowFileRepository.java | 2 +-
.../WriteAheadFlowFileRepository.java | 2 +-
.../controller/TestFileSystemSwapManager.java | 6 +-
.../repository/TestStandardProcessSession.java | 9 +-
.../TestWriteAheadFlowFileRepository.java | 2 +-
26 files changed, 1003 insertions(+), 838 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/b8c51dc3/nifi-api/src/main/java/org/apache/nifi/controller/FlowFileQueue.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/FlowFileQueue.java b/nifi-api/src/main/java/org/apache/nifi/controller/FlowFileQueue.java
deleted file mode 100644
index e1baeb7..0000000
--- a/nifi-api/src/main/java/org/apache/nifi/controller/FlowFileQueue.java
+++ /dev/null
@@ -1,203 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.controller;
-
-import java.util.Collection;
-import java.util.List;
-import java.util.Queue;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.nifi.controller.repository.FlowFileRecord;
-import org.apache.nifi.flowfile.FlowFilePrioritizer;
-import org.apache.nifi.processor.FlowFileFilter;
-import org.apache.nifi.processor.QueueSize;
-
-public interface FlowFileQueue {
-
- /**
- * @return the unique identifier for this FlowFileQueue
- */
- String getIdentifier();
-
- /**
- * @return list of processing priorities for this queue
- */
- List<FlowFilePrioritizer> getPriorities();
-
- /**
- * @return the minimum number of FlowFiles that must be present in order for
- * FlowFiles to begin being swapped out of the queue
- */
- int getSwapThreshold();
-
- /**
- * Resets the comparator used by this queue to maintain order.
- *
- * @param newPriorities the ordered list of prioritizers to use to determine
- * order within this queue.
- * @throws NullPointerException if arg is null
- */
- void setPriorities(List<FlowFilePrioritizer> newPriorities);
-
- /**
- * Establishes this queue's preferred maximum work load.
- *
- * @param maxQueueSize the maximum number of flow files this processor
- * recommends having in its work queue at any one time
- */
- void setBackPressureObjectThreshold(long maxQueueSize);
-
- /**
- * @return maximum number of flow files that should be queued up at any one
- * time
- */
- long getBackPressureObjectThreshold();
-
- /**
- * @param maxDataSize Establishes this queue's preferred maximum data size.
- */
- void setBackPressureDataSizeThreshold(String maxDataSize);
-
- /**
- * @return maximum data size that should be queued up at any one time
- */
- String getBackPressureDataSizeThreshold();
-
- QueueSize size();
-
- /**
- * @return total size in bytes of the queue flow file's content
- */
- long contentSize();
-
- /**
- * @return true if no items queue; false otherwise
- */
- boolean isEmpty();
-
- /**
- * @return true if the active queue is empty; false otherwise. The Active
- * queue contains those FlowFiles that can be processed immediately and does
- * not include those FlowFiles that have been swapped out or are currently
- * being processed
- */
- boolean isActiveQueueEmpty();
-
- QueueSize getActiveQueueSize();
-
- /**
- * Returns a QueueSize that represents all FlowFiles that are 'unacknowledged'. A FlowFile
- * is considered to be unacknowledged if it has been pulled from the queue by some component
- * but the session that pulled the FlowFile has not yet been committed or rolled back.
- *
- * @return a QueueSize that represents all FlowFiles that are 'unacknowledged'.
- */
- QueueSize getUnacknowledgedQueueSize();
-
- void acknowledge(FlowFileRecord flowFile);
-
- void acknowledge(Collection<FlowFileRecord> flowFiles);
-
- /**
- * @return true if maximum queue size has been reached or exceeded; false
- * otherwise
- */
- boolean isFull();
-
- /**
- * places the given file into the queue
- *
- * @param file to place into queue
- */
- void put(FlowFileRecord file);
-
- /**
- * places the given files into the queue
- *
- * @param files to place into queue
- */
- void putAll(Collection<FlowFileRecord> files);
-
- /**
- * Removes all records from the internal swap queue and returns them.
- *
- * @return all removed records from internal swap queue
- */
- List<FlowFileRecord> pollSwappableRecords();
-
- /**
- * Restores the records from swap space into this queue, adding the records
- * that have expired to the given set instead of enqueuing them.
- *
- * @param records that were swapped in
- */
- void putSwappedRecords(Collection<FlowFileRecord> records);
-
- /**
- * Updates the internal counters of how much data is queued, based on
- * swapped data that is being restored.
- *
- * @param numRecords count of records swapped in
- * @param contentSize total size of records being swapped in
- */
- void incrementSwapCount(int numRecords, long contentSize);
-
- /**
- * @return the number of FlowFiles that are enqueued and not swapped
- */
- int unswappedSize();
-
- int getSwapRecordCount();
-
- int getSwapQueueSize();
-
- /**
- * @param expiredRecords expired records
- * @return the next flow file on the queue; null if empty
- */
- FlowFileRecord poll(Set<FlowFileRecord> expiredRecords);
-
- /**
- * @param maxResults limits how many results can be polled
- * @param expiredRecords for expired records
- * @return the next flow files on the queue up to the max results; null if
- * empty
- */
- List<FlowFileRecord> poll(int maxResults, Set<FlowFileRecord> expiredRecords);
-
- /**
- * Drains flow files from the given source queue into the given destination
- * list.
- *
- * @param sourceQueue queue to drain from
- * @param destination Collection to drain to
- * @param maxResults max number to drain
- * @param expiredRecords for expired records
- * @return size (bytes) of flow files drained from queue
- */
- long drainQueue(Queue<FlowFileRecord> sourceQueue, List<FlowFileRecord> destination, int maxResults, Set<FlowFileRecord> expiredRecords);
-
- List<FlowFileRecord> poll(FlowFileFilter filter, Set<FlowFileRecord> expiredRecords);
-
- String getFlowFileExpiration();
-
- int getFlowFileExpiration(TimeUnit timeUnit);
-
- void setFlowFileExpiration(String flowExpirationPeriod);
-
-}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b8c51dc3/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileState.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileState.java b/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileState.java
new file mode 100644
index 0000000..3f16d00
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileState.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.controller.queue;
+
+/**
+ * Represents the state that a Drop FlowFile request is in
+ */
+public enum DropFlowFileState {
+
+ WAITING_FOR_LOCK("Waiting for Destination Component to complete its action"),
+ DROPPING_ACTIVE_FLOWFILES("Dropping FlowFiles from queue"),
+ COMPLETE("Completed Successfully"),
+ FAILURE("Failed");
+
+ private final String description;
+
+ private DropFlowFileState(final String description) {
+ this.description = description;
+ }
+
+ @Override
+ public String toString() {
+ return description;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b8c51dc3/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileStatus.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileStatus.java b/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileStatus.java
new file mode 100644
index 0000000..b216608
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileStatus.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.controller.queue;
+
+/**
+ * Represents the status of a Drop FlowFile Request that has been issued to
+ * a {@link FlowFileQueue}. When a queue is requested to drop its FlowFiles,
+ * that process may be rather lengthy in the case of a poorly behaving
+ * FlowFileRepository or if the destination Processor is polling from the
+ * queue using a filter that is misbehaving. As a result, the dropping of
+ * FlowFiles is performed asynchronously.
+ *
+ * This status object provides information about how far along in the process
+ * we currently are and information about the success or failure of the
+ * operation.
+ */
+public interface DropFlowFileStatus {
+
+ /**
+ * @return the identifier of the request to drop FlowFiles from the queue
+ */
+ String getRequestIdentifier();
+
+ /**
+ * @return the date/time (in milliseconds since epoch) at which the request to
+ * drop the FlowFiles from a queue was submitted
+ */
+ long getRequestSubmissionTime();
+
+ /**
+ * @return the size of the queue when the drop request was issued or <code>null</code> if
+ * it is not yet known, which can happen if the {@link DropFlowFileState} is
+ * {@link DropFlowFileState#WAITING_FOR_LOCK}.
+ */
+ QueueSize getOriginalSize();
+
+ /**
+ * @return the current size of the queue or <code>null</code> if it is not yet known
+ */
+ QueueSize getCurrentSize();
+
+ /**
+ * @return the current state of the operation
+ */
+ DropFlowFileState getState();
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b8c51dc3/nifi-api/src/main/java/org/apache/nifi/controller/queue/FlowFileQueue.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/queue/FlowFileQueue.java b/nifi-api/src/main/java/org/apache/nifi/controller/queue/FlowFileQueue.java
new file mode 100644
index 0000000..31f17e0
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/queue/FlowFileQueue.java
@@ -0,0 +1,256 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller.queue;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Queue;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.nifi.controller.repository.FlowFileRecord;
+import org.apache.nifi.flowfile.FlowFilePrioritizer;
+import org.apache.nifi.processor.FlowFileFilter;
+
+public interface FlowFileQueue {
+
+ /**
+ * @return the unique identifier for this FlowFileQueue
+ */
+ String getIdentifier();
+
+ /**
+ * @return list of processing priorities for this queue
+ */
+ List<FlowFilePrioritizer> getPriorities();
+
+ /**
+ * Reads any Swap Files that belong to this queue and increments counts so that the size
+ * of the queue will reflect the size of all FlowFiles regardless of whether or not they are
+ * swapped out. This will be called only during NiFi startup as an initialization step. This
+ * method is then responsible for returning the largest ID of any FlowFile that is swapped
+ * out, or <code>null</code> if no FlowFiles are swapped out for this queue.
+ *
+ * @return the largest ID of any FlowFile that is swapped out for this queue, or <code>null</code> if
+ * no FlowFiles are swapped out for this queue.
+ */
+ Long recoverSwappedFlowFiles();
+
+ /**
+ * Destroys any Swap Files that exist for this queue without updating the FlowFile Repository
+ * or Provenance Repository. This is done only on startup in the case of non-persistent
+ * repositories. In the case of non-persistent repositories, we may still have Swap Files because
+ * we may still need to overflow the FlowFiles from heap onto disk, even though we don't want to keep
+ * the FlowFiles on restart.
+ */
+ void purgeSwapFiles();
+
+ /**
+ * @return the minimum number of FlowFiles that must be present in order for
+ * FlowFiles to begin being swapped out of the queue
+ */
+ // TODO: REMOVE THIS.
+ int getSwapThreshold();
+
+ /**
+ * Resets the comparator used by this queue to maintain order.
+ *
+ * @param newPriorities the ordered list of prioritizers to use to determine
+ * order within this queue.
+ * @throws NullPointerException if arg is null
+ */
+ void setPriorities(List<FlowFilePrioritizer> newPriorities);
+
+ /**
+ * Establishes this queue's preferred maximum work load.
+ *
+ * @param maxQueueSize the maximum number of flow files this processor
+ * recommends having in its work queue at any one time
+ */
+ void setBackPressureObjectThreshold(long maxQueueSize);
+
+ /**
+ * @return maximum number of flow files that should be queued up at any one
+ * time
+ */
+ long getBackPressureObjectThreshold();
+
+ /**
+ * @param maxDataSize Establishes this queue's preferred maximum data size.
+ */
+ void setBackPressureDataSizeThreshold(String maxDataSize);
+
+ /**
+ * @return maximum data size that should be queued up at any one time
+ */
+ String getBackPressureDataSizeThreshold();
+
+ QueueSize size();
+
+ /**
+ * @return true if no items queue; false otherwise
+ */
+ boolean isEmpty();
+
+ /**
+ * @return true if the active queue is empty; false otherwise. The Active
+ * queue contains those FlowFiles that can be processed immediately and does
+ * not include those FlowFiles that have been swapped out or are currently
+ * being processed
+ */
+ // TODO: REMOVE?
+ boolean isActiveQueueEmpty();
+
+ // TODO: REMOVE?
+ QueueSize getActiveQueueSize();
+
+ /**
+ * Returns a QueueSize that represents all FlowFiles that are 'unacknowledged'. A FlowFile
+ * is considered to be unacknowledged if it has been pulled from the queue by some component
+ * but the session that pulled the FlowFile has not yet been committed or rolled back.
+ *
+ * @return a QueueSize that represents all FlowFiles that are 'unacknowledged'.
+ */
+ QueueSize getUnacknowledgedQueueSize();
+
+ void acknowledge(FlowFileRecord flowFile);
+
+ void acknowledge(Collection<FlowFileRecord> flowFiles);
+
+ /**
+ * @return true if maximum queue size has been reached or exceeded; false
+ * otherwise
+ */
+ boolean isFull();
+
+ /**
+ * places the given file into the queue
+ *
+ * @param file to place into queue
+ */
+ void put(FlowFileRecord file);
+
+ /**
+ * places the given files into the queue
+ *
+ * @param files to place into queue
+ */
+ void putAll(Collection<FlowFileRecord> files);
+
+ /**
+ * Removes all records from the internal swap queue and returns them.
+ *
+ * @return all removed records from internal swap queue
+ */
+ // TODO: REMOVE THIS?
+ List<FlowFileRecord> pollSwappableRecords();
+
+ /**
+ * Restores the records from swap space into this queue, adding the records
+ * that have expired to the given set instead of enqueuing them.
+ *
+ * @param records that were swapped in
+ */
+ // TODO: REMOVE THIS?
+ void putSwappedRecords(Collection<FlowFileRecord> records);
+
+ /**
+ * Updates the internal counters of how much data is queued, based on
+ * swapped data that is being restored.
+ *
+ * @param numRecords count of records swapped in
+ * @param contentSize total size of records being swapped in
+ */
+ // TODO: REMOVE THIS?
+ void incrementSwapCount(int numRecords, long contentSize);
+
+ /**
+ * @return the number of FlowFiles that are enqueued and not swapped
+ */
+ // TODO: REMOVE THIS?
+ int unswappedSize();
+
+ // TODO: REMOVE THIS?
+ int getSwapRecordCount();
+
+ // TODO: REMOVE THIS?
+ int getSwapQueueSize();
+
+ /**
+ * @param expiredRecords expired records
+ * @return the next flow file on the queue; null if empty
+ */
+ FlowFileRecord poll(Set<FlowFileRecord> expiredRecords);
+
+ /**
+ * @param maxResults limits how many results can be polled
+ * @param expiredRecords for expired records
+ * @return the next flow files on the queue up to the max results; null if
+ * empty
+ */
+ List<FlowFileRecord> poll(int maxResults, Set<FlowFileRecord> expiredRecords);
+
+ /**
+ * Drains flow files from the given source queue into the given destination
+ * list.
+ *
+ * @param sourceQueue queue to drain from
+ * @param destination Collection to drain to
+ * @param maxResults max number to drain
+ * @param expiredRecords for expired records
+ * @return size (bytes) of flow files drained from queue
+ */
+ long drainQueue(Queue<FlowFileRecord> sourceQueue, List<FlowFileRecord> destination, int maxResults, Set<FlowFileRecord> expiredRecords);
+
+ List<FlowFileRecord> poll(FlowFileFilter filter, Set<FlowFileRecord> expiredRecords);
+
+ String getFlowFileExpiration();
+
+ int getFlowFileExpiration(TimeUnit timeUnit);
+
+ void setFlowFileExpiration(String flowExpirationPeriod);
+
+ /**
+ * Initiates a request to drop all FlowFiles in this queue. This method returns
+ * a DropFlowFileStatus that can be used to determine the current state of the request.
+ * Additionally, the DropFlowFileStatus provides a request identifier that can then be
+ * passed to the {@link #getDropFlowFileStatus(String)} and {@link #cancelDropFlowFileStatus(String)}
+ * methods in order to obtain the status later or cancel a request
+ *
+ * @return the status of the drop request.
+ */
+ DropFlowFileStatus dropFlowFiles();
+
+ /**
+ * Returns the current status of a Drop FlowFile Request that was initiated via the
+ * {@link #dropFlowFiles()} method that has the given identifier
+ *
+ * @param requestIdentifier the identifier of the Drop FlowFile Request
+ * @return the status for the request with the given identifier, or <code>null</code> if no
+ * request status exists with that identifier
+ */
+ DropFlowFileStatus getDropFlowFileStatus(String requestIdentifier);
+
+ /**
+ * Cancels the request to drop FlowFiles that has the given identifier
+ *
+ * @param requestIdentifier the identifier of the Drop FlowFile Request
+ * @return <code>true</code> if the request was canceled, <code>false</code> if the request has
+ * already completed or is not known
+ */
+ boolean cancelDropFlowFileRequest(String requestIdentifier);
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b8c51dc3/nifi-api/src/main/java/org/apache/nifi/controller/queue/QueueSize.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/queue/QueueSize.java b/nifi-api/src/main/java/org/apache/nifi/controller/queue/QueueSize.java
new file mode 100644
index 0000000..42d8416
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/queue/QueueSize.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller.queue;
+
+/**
+ *
+ */
+public class QueueSize {
+
+ private final int objectCount;
+ private final long totalSizeBytes;
+
+ public QueueSize(final int numberObjects, final long totalSizeBytes) {
+ if (numberObjects < 0 || totalSizeBytes < 0) {
+ throw new IllegalArgumentException();
+ }
+ objectCount = numberObjects;
+ this.totalSizeBytes = totalSizeBytes;
+ }
+
+ /**
+ * @return number of objects present on the queue
+ */
+ public int getObjectCount() {
+ return objectCount;
+ }
+
+ /**
+ * @return total size in bytes of the content for the data on the queue
+ */
+ public long getByteCount() {
+ return totalSizeBytes;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b8c51dc3/nifi-api/src/main/java/org/apache/nifi/controller/repository/FlowFileRepository.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/repository/FlowFileRepository.java b/nifi-api/src/main/java/org/apache/nifi/controller/repository/FlowFileRepository.java
index 58fc6b3..906cbe2 100644
--- a/nifi-api/src/main/java/org/apache/nifi/controller/repository/FlowFileRepository.java
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/repository/FlowFileRepository.java
@@ -21,7 +21,7 @@ import java.io.IOException;
import java.util.Collection;
import java.util.List;
-import org.apache.nifi.controller.FlowFileQueue;
+import org.apache.nifi.controller.queue.FlowFileQueue;
import org.apache.nifi.controller.repository.claim.ResourceClaimManager;
/**
http://git-wip-us.apache.org/repos/asf/nifi/blob/b8c51dc3/nifi-api/src/main/java/org/apache/nifi/controller/repository/FlowFileSwapManager.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/repository/FlowFileSwapManager.java b/nifi-api/src/main/java/org/apache/nifi/controller/repository/FlowFileSwapManager.java
index 2e5be11..57e9186 100644
--- a/nifi-api/src/main/java/org/apache/nifi/controller/repository/FlowFileSwapManager.java
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/repository/FlowFileSwapManager.java
@@ -16,8 +16,11 @@
*/
package org.apache.nifi.controller.repository;
-import org.apache.nifi.controller.repository.claim.ResourceClaimManager;
-import org.apache.nifi.events.EventReporter;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.nifi.controller.queue.FlowFileQueue;
+import org.apache.nifi.controller.queue.QueueSize;
/**
* Defines a mechanism by which FlowFiles can be move into external storage or
@@ -26,38 +29,81 @@ import org.apache.nifi.events.EventReporter;
public interface FlowFileSwapManager {
/**
- * Starts the Manager's background threads to start swapping FlowFiles in
- * and out of memory
+ * Initializes the Swap Manager, providing a {@link SwapManagerInitializationContext} so that the
+ * Swap Manager has access to all of the components necessary to perform its functions
*
- * @param flowFileRepository the FlowFileRepository that must be notified of
- * any swapping in or out of FlowFiles
- * @param queueProvider the provider of FlowFileQueue's so that FlowFiles
- * can be obtained and restored
- * @param claimManager the ContentClaimManager to use for interacting with
- * Content Claims
- * @param reporter the EventReporter that can be used for notifying users of
- * important events
+ * @param initializationContext the context the provides the swap manager with access to the
+ * resources that it needs to perform its functions
*/
- void start(FlowFileRepository flowFileRepository, QueueProvider queueProvider, ResourceClaimManager claimManager, EventReporter reporter);
+ void initialize(SwapManagerInitializationContext initializationContext);
/**
- * Shuts down the manager
+ * Swaps out the given FlowFiles that belong to the queue with the given identifier.
+ *
+ * @param flowFiles the FlowFiles to swap out to external storage
+ * @param flowFileQueue the queue that the FlowFiles belong to
+ * @return the location of the externally stored swap file
+ *
+ * @throws IOException if unable to swap the FlowFiles out
*/
- void shutdown();
+ String swapOut(List<FlowFileRecord> flowFiles, FlowFileQueue flowFileQueue) throws IOException;
/**
- * Removes all Swap information, permanently destroying any FlowFiles that
- * have been swapped out
+ * Recovers the SwapFiles from the swap file that lives at the given location. This action
+ * provides a view of the FlowFiles but does not actively swap them in, meaning that the swap file
+ * at the given location remains in that location and the FlowFile Repository is not updated.
+ *
+ * @param swapLocation the location of hte swap file
+ * @param flowFileQueue the queue that the FlowFiles belong to
+ * @return the FlowFiles that live at the given swap location
+ *
+ * @throws IOException if unable to recover the FlowFiles from the given location
*/
- void purge();
+ List<FlowFileRecord> peek(String swapLocation, final FlowFileQueue flowFileQueue) throws IOException;
/**
- * Notifies FlowFile queues of the number of FlowFiles and content size of
- * all FlowFiles that are currently swapped out
+ * Recovers the FlowFiles from the swap file that lives at the given location and belongs
+ * to the FlowFile Queue with the given identifier. The FlowFile Repository is then updated
+ * and the swap file is permanently removed from the external storage
+ *
+ * @param swapLocation the location of the swap file
+ * @param flowFileQueue the queue to which the FlowFiles belong
+ *
+ * @return the FlowFiles that are stored in the given location
+ *
+ * @throws IOException if unable to recover the FlowFiles from the given location or update the
+ * FlowFileRepository
+ */
+ List<FlowFileRecord> swapIn(String swapLocation, FlowFileQueue flowFileQueue) throws IOException;
+
+ /**
+ * Determines swap files that exist for the given FlowFileQueue
+ *
+ * @param flowFileQueue the queue for which the FlowFiles should be recovered
*
- * @param connectionProvider provider
- * @param claimManager manager
- * @return how many flowfiles have been recovered
+ * @return all swap locations that have been identified for the given queue, in the order that they should
+ * be swapped back in
*/
- long recoverSwappedFlowFiles(QueueProvider connectionProvider, ResourceClaimManager claimManager);
+ List<String> recoverSwapLocations(FlowFileQueue flowFileQueue) throws IOException;
+
+ /**
+ * Determines how many FlowFiles and the size of the FlowFiles that are swapped out at the given location
+ *
+ * @param swapLocation the location of the swap file
+ * @return the QueueSize representing the number of FlowFiles and total size of the FlowFiles that are swapped out
+ */
+ QueueSize getSwapSize(String swapLocation) throws IOException;
+
+ /**
+ * Returns the maximum record id of the FlowFiles stored at the given swap location
+ *
+ * @param swapLocation the swap location to read id's from
+ * @return the max record id of any FlowFile in the swap location, or null if no record ID's can be found
+ */
+ Long getMaxRecordId(String swapLocation) throws IOException;
+
+ /**
+ * Purge all known Swap Files without updating FlowFileRepository or Provenance Repository
+ */
+ void purge();
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b8c51dc3/nifi-api/src/main/java/org/apache/nifi/controller/repository/QueueProvider.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/repository/QueueProvider.java b/nifi-api/src/main/java/org/apache/nifi/controller/repository/QueueProvider.java
index fcb516d..95d9f2e 100644
--- a/nifi-api/src/main/java/org/apache/nifi/controller/repository/QueueProvider.java
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/repository/QueueProvider.java
@@ -18,7 +18,7 @@ package org.apache.nifi.controller.repository;
import java.util.Collection;
-import org.apache.nifi.controller.FlowFileQueue;
+import org.apache.nifi.controller.queue.FlowFileQueue;
/**
* Provides a collection of <code>FlowFileQueue</code>s that represents all
http://git-wip-us.apache.org/repos/asf/nifi/blob/b8c51dc3/nifi-api/src/main/java/org/apache/nifi/controller/repository/RepositoryRecord.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/repository/RepositoryRecord.java b/nifi-api/src/main/java/org/apache/nifi/controller/repository/RepositoryRecord.java
index 40d44a8..09202c0 100644
--- a/nifi-api/src/main/java/org/apache/nifi/controller/repository/RepositoryRecord.java
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/repository/RepositoryRecord.java
@@ -16,7 +16,7 @@
*/
package org.apache.nifi.controller.repository;
-import org.apache.nifi.controller.FlowFileQueue;
+import org.apache.nifi.controller.queue.FlowFileQueue;
import org.apache.nifi.controller.repository.claim.ContentClaim;
/**
http://git-wip-us.apache.org/repos/asf/nifi/blob/b8c51dc3/nifi-api/src/main/java/org/apache/nifi/controller/repository/SwapManagerInitializationContext.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/repository/SwapManagerInitializationContext.java b/nifi-api/src/main/java/org/apache/nifi/controller/repository/SwapManagerInitializationContext.java
new file mode 100644
index 0000000..564d5ec
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/repository/SwapManagerInitializationContext.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.controller.repository;
+
+import org.apache.nifi.controller.repository.claim.ResourceClaimManager;
+import org.apache.nifi.events.EventReporter;
+
+public interface SwapManagerInitializationContext {
+
+ /**
+ * @return the {@link FlowFileRepository} that should be updated when FlowFiles are swapped in and out
+ */
+ FlowFileRepository getFlowFileRepository();
+
+
+ /**
+ * @return the {@link ResourceClaimManager} that is necessary to provide to the FlowFileRepository when
+ * performing swapping actions
+ */
+ ResourceClaimManager getResourceClaimManager();
+
+ /**
+ * @return an {@link EventReporter} that can be used to report events to users
+ */
+ EventReporter getEventReporter();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b8c51dc3/nifi-api/src/main/java/org/apache/nifi/processor/ProcessSession.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/ProcessSession.java b/nifi-api/src/main/java/org/apache/nifi/processor/ProcessSession.java
index ed46d68..ebd56a9 100644
--- a/nifi-api/src/main/java/org/apache/nifi/processor/ProcessSession.java
+++ b/nifi-api/src/main/java/org/apache/nifi/processor/ProcessSession.java
@@ -25,6 +25,7 @@ import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
+import org.apache.nifi.controller.queue.QueueSize;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.processor.exception.FlowFileAccessException;
import org.apache.nifi.processor.exception.FlowFileHandlingException;
http://git-wip-us.apache.org/repos/asf/nifi/blob/b8c51dc3/nifi-api/src/main/java/org/apache/nifi/processor/QueueSize.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/QueueSize.java b/nifi-api/src/main/java/org/apache/nifi/processor/QueueSize.java
deleted file mode 100644
index c3c2ccc..0000000
--- a/nifi-api/src/main/java/org/apache/nifi/processor/QueueSize.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.processor;
-
-/**
- *
- */
-public class QueueSize {
-
- private final int objectCount;
- private final long totalSizeBytes;
-
- public QueueSize(final int numberObjects, final long totalSizeBytes) {
- if (numberObjects < 0 || totalSizeBytes < 0) {
- throw new IllegalArgumentException();
- }
- objectCount = numberObjects;
- this.totalSizeBytes = totalSizeBytes;
- }
-
- /**
- * @return number of objects present on the queue
- */
- public int getObjectCount() {
- return objectCount;
- }
-
- /**
- * @return total size in bytes of the content for the data on the queue
- */
- public long getByteCount() {
- return totalSizeBytes;
- }
-}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b8c51dc3/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/connectable/Connection.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/connectable/Connection.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/connectable/Connection.java
index 0a0089d..2e66905 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/connectable/Connection.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/connectable/Connection.java
@@ -20,7 +20,7 @@ import java.util.Collection;
import java.util.List;
import java.util.Set;
-import org.apache.nifi.controller.FlowFileQueue;
+import org.apache.nifi.controller.queue.FlowFileQueue;
import org.apache.nifi.controller.repository.FlowFileRecord;
import org.apache.nifi.groups.ProcessGroup;
import org.apache.nifi.processor.FlowFileFilter;
http://git-wip-us.apache.org/repos/asf/nifi/blob/b8c51dc3/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
index f47ea2f..df356fd 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
@@ -16,6 +16,7 @@
*/
package org.apache.nifi.controller;
+import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collection;
@@ -34,21 +35,25 @@ import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.nifi.connectable.Connection;
+import org.apache.nifi.controller.queue.DropFlowFileStatus;
+import org.apache.nifi.controller.queue.FlowFileQueue;
+import org.apache.nifi.controller.queue.QueueSize;
import org.apache.nifi.controller.repository.FlowFileRecord;
+import org.apache.nifi.controller.repository.FlowFileSwapManager;
import org.apache.nifi.controller.repository.claim.ContentClaim;
+import org.apache.nifi.events.EventReporter;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.flowfile.FlowFilePrioritizer;
import org.apache.nifi.processor.DataUnit;
import org.apache.nifi.processor.FlowFileFilter;
import org.apache.nifi.processor.FlowFileFilter.FlowFileFilterResult;
-import org.apache.nifi.processor.QueueSize;
+import org.apache.nifi.reporting.Severity;
import org.apache.nifi.scheduling.SchedulingStrategy;
import org.apache.nifi.util.FormatUtils;
import org.apache.nifi.util.concurrency.TimedLock;
import org.apache.nifi.util.timebuffer.LongEntityAccess;
import org.apache.nifi.util.timebuffer.TimedBuffer;
import org.apache.nifi.util.timebuffer.TimestampedLong;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -84,12 +89,14 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
private boolean swapMode = false;
private long maximumQueueObjectCount;
+ private final EventReporter eventReporter;
private final AtomicLong flowFileExpirationMillis;
private final Connection connection;
private final AtomicReference<String> flowFileExpirationPeriod;
private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true);
private final List<FlowFilePrioritizer> priorities;
private final int swapThreshold;
+ private final FlowFileSwapManager swapManager;
private final TimedLock readLock;
private final TimedLock writeLock;
private final String identifier;
@@ -101,7 +108,8 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
// SCHEDULER CANNOT BE NOTIFIED OF EVENTS WITH THE WRITE LOCK HELD! DOING SO WILL RESULT IN A DEADLOCK!
private final ProcessScheduler scheduler;
- public StandardFlowFileQueue(final String identifier, final Connection connection, final ProcessScheduler scheduler, final int swapThreshold) {
+ public StandardFlowFileQueue(final String identifier, final Connection connection, final ProcessScheduler scheduler, final FlowFileSwapManager swapManager, final EventReporter eventReporter,
+ final int swapThreshold) {
activeQueue = new PriorityQueue<>(20, new Prioritizer(new ArrayList<FlowFilePrioritizer>()));
priorities = new ArrayList<>();
maximumQueueObjectCount = 0L;
@@ -110,6 +118,8 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
flowFileExpirationMillis = new AtomicLong(0);
flowFileExpirationPeriod = new AtomicReference<>("0 mins");
swapQueue = new ArrayList<>();
+ this.eventReporter = eventReporter;
+ this.swapManager = swapManager;
this.identifier = identifier;
this.swapThreshold = swapThreshold;
@@ -233,21 +243,6 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
}
@Override
- public long contentSize() {
- readLock.lock();
- try {
- final PreFetch prefetch = preFetchRef.get();
- if (prefetch == null) {
- return activeQueueContentSize + swappedContentSize + unacknowledgedSizeRef.get().getObjectCount();
- } else {
- return activeQueueContentSize + swappedContentSize + unacknowledgedSizeRef.get().getObjectCount() + prefetch.size().getByteCount();
- }
- } finally {
- readLock.unlock("getContentSize");
- }
- }
-
- @Override
public boolean isEmpty() {
readLock.lock();
try {
@@ -945,11 +940,88 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
this.flowFileExpirationMillis.set(millis);
}
+
+ @Override
+ public void purgeSwapFiles() {
+ swapManager.purge();
+ }
+
+ @Override
+ public Long recoverSwappedFlowFiles() {
+ int swapFlowFileCount = 0;
+ long swapByteCount = 0L;
+ Long maxId = null;
+
+ writeLock.lock();
+ try {
+ final List<String> swapLocations;
+ try {
+ swapLocations = swapManager.recoverSwapLocations(this);
+ } catch (final IOException ioe) {
+ logger.error("Failed to determine whether or not any Swap Files exist for FlowFile Queue {}", getIdentifier());
+ logger.error("", ioe);
+ if (eventReporter != null) {
+ eventReporter.reportEvent(Severity.ERROR, "FlowFile Swapping", "Failed to determine whether or not any Swap Files exist for FlowFile Queue " +
+ getIdentifier() + "; see logs for more detials");
+ }
+ return null;
+ }
+
+ for (final String swapLocation : swapLocations) {
+ try {
+ final QueueSize queueSize = swapManager.getSwapSize(swapLocation);
+ final Long maxSwapRecordId = swapManager.getMaxRecordId(swapLocation);
+ if (maxSwapRecordId != null) {
+ if (maxId == null || maxSwapRecordId > maxId) {
+ maxId = maxSwapRecordId;
+ }
+ }
+
+ swapFlowFileCount += queueSize.getObjectCount();
+ swapByteCount += queueSize.getByteCount();
+ } catch (final IOException ioe) {
+ logger.error("Failed to recover FlowFiles from Swap File {}; the file appears to be corrupt", swapLocation, ioe.toString());
+ logger.error("", ioe);
+ if (eventReporter != null) {
+ eventReporter.reportEvent(Severity.ERROR, "FlowFile Swapping", "Failed to recover FlowFiles from Swap File " + swapLocation +
+ "; the file appears to be corrupt. See logs for more details");
+ }
+ }
+ }
+
+ this.swappedRecordCount = swapFlowFileCount;
+ this.swappedContentSize = swapByteCount;
+ } finally {
+ writeLock.unlock("Recover Swap Files");
+ }
+
+ return maxId;
+ }
+
+
@Override
public String toString() {
return "FlowFileQueue[id=" + identifier + "]";
}
+ @Override
+ public DropFlowFileStatus dropFlowFiles() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public boolean cancelDropFlowFileRequest(String requestIdentifier) {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ @Override
+ public DropFlowFileStatus getDropFlowFileStatus(String requestIdentifier) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
/**
* Lock the queue so that other threads are unable to interact with the
* queue
http://git-wip-us.apache.org/repos/asf/nifi/blob/b8c51dc3/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/connectable/StandardConnection.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/connectable/StandardConnection.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/connectable/StandardConnection.java
index ad556e2..f0a6d8a 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/connectable/StandardConnection.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/connectable/StandardConnection.java
@@ -26,18 +26,19 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
-import org.apache.nifi.controller.FlowFileQueue;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.nifi.controller.ProcessScheduler;
import org.apache.nifi.controller.StandardFlowFileQueue;
+import org.apache.nifi.controller.queue.FlowFileQueue;
import org.apache.nifi.controller.repository.FlowFileRecord;
+import org.apache.nifi.controller.repository.FlowFileSwapManager;
+import org.apache.nifi.events.EventReporter;
import org.apache.nifi.groups.ProcessGroup;
import org.apache.nifi.processor.FlowFileFilter;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.util.NiFiProperties;
-import org.apache.commons.lang3.builder.EqualsBuilder;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-
/**
* Models a connection between connectable components. A connection may contain one or more relationships that map the source component to the destination component.
*/
@@ -65,7 +66,7 @@ public final class StandardConnection implements Connection {
destination = new AtomicReference<>(builder.destination);
relationships = new AtomicReference<>(Collections.unmodifiableCollection(builder.relationships));
scheduler = builder.scheduler;
- flowFileQueue = new StandardFlowFileQueue(id, this, scheduler, NiFiProperties.getInstance().getQueueSwapThreshold());
+ flowFileQueue = new StandardFlowFileQueue(id, this, scheduler, builder.swapManager, builder.eventReporter, NiFiProperties.getInstance().getQueueSwapThreshold());
hashCode = new HashCodeBuilder(7, 67).append(id).toHashCode();
}
@@ -259,6 +260,8 @@ public final class StandardConnection implements Connection {
private Connectable source;
private Connectable destination;
private Collection<Relationship> relationships;
+ private FlowFileSwapManager swapManager;
+ private EventReporter eventReporter;
public Builder(final ProcessScheduler scheduler) {
this.scheduler = scheduler;
@@ -305,6 +308,16 @@ public final class StandardConnection implements Connection {
return this;
}
+ public Builder swapManager(final FlowFileSwapManager swapManager) {
+ this.swapManager = swapManager;
+ return this;
+ }
+
+ public Builder eventReporter(final EventReporter eventReporter) {
+ this.eventReporter = eventReporter;
+ return this;
+ }
+
public StandardConnection build() {
if (source == null) {
throw new IllegalStateException("Cannot build a Connection without a Source");
@@ -312,6 +325,9 @@ public final class StandardConnection implements Connection {
if (destination == null) {
throw new IllegalStateException("Cannot build a Connection without a Destination");
}
+ if (swapManager == null) {
+ throw new IllegalStateException("Cannot build a Connection without a FlowFileSwapManager");
+ }
if (relationships == null) {
relationships = new ArrayList<>();
[30/30] nifi git commit: NIFI-730: Completed merging of master
Posted by ma...@apache.org.
NIFI-730: Completed merging of master
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/dbf0c789
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/dbf0c789
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/dbf0c789
Branch: refs/heads/master
Commit: dbf0c7893fef964bfbb3a4c039c756396587ce12
Parents: e4cebba
Author: Mark Payne <ma...@hotmail.com>
Authored: Mon Nov 2 14:31:28 2015 -0500
Committer: Mark Payne <ma...@hotmail.com>
Committed: Mon Nov 2 14:31:28 2015 -0500
----------------------------------------------------------------------
.../apache/nifi/controller/FlowController.java | 73 +++++++++++---------
1 file changed, 40 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/dbf0c789/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java
index 701459c..a3246fd 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java
@@ -16,39 +16,11 @@
*/
package org.apache.nifi.controller;
-import static java.util.Objects.requireNonNull;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import javax.net.ssl.SSLContext;
-
+import com.sun.jersey.api.client.ClientHandlerException;
+import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.action.Action;
+import org.apache.nifi.admin.service.AuditService;
+import org.apache.nifi.admin.service.UserService;
import org.apache.nifi.annotation.lifecycle.OnAdded;
import org.apache.nifi.annotation.lifecycle.OnRemoved;
import org.apache.nifi.annotation.lifecycle.OnShutdown;
@@ -141,6 +113,7 @@ import org.apache.nifi.groups.ProcessGroup;
import org.apache.nifi.groups.RemoteProcessGroup;
import org.apache.nifi.groups.RemoteProcessGroupPortDescriptor;
import org.apache.nifi.groups.StandardProcessGroup;
+import org.apache.nifi.history.History;
import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.logging.ControllerServiceLogObserver;
import org.apache.nifi.logging.LogLevel;
@@ -203,7 +176,36 @@ import org.apache.nifi.web.api.dto.status.StatusHistoryDTO;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.sun.jersey.api.client.ClientHandlerException;
+import javax.net.ssl.SSLContext;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import static java.util.Objects.requireNonNull;
public class FlowController implements EventAccess, ControllerServiceProvider, ReportingTaskProvider, Heartbeater, QueueProvider {
@@ -345,11 +347,13 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
final FlowFileEventRepository flowFileEventRepo,
final NiFiProperties properties,
final UserService userService,
+ final AuditService auditService,
final StringEncryptor encryptor) {
return new FlowController(
flowFileEventRepo,
properties,
userService,
+ auditService,
encryptor,
/* configuredForClustering */ false,
/* NodeProtocolSender */ null);
@@ -359,12 +363,14 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
final FlowFileEventRepository flowFileEventRepo,
final NiFiProperties properties,
final UserService userService,
+ final AuditService auditService,
final StringEncryptor encryptor,
final NodeProtocolSender protocolSender) {
final FlowController flowController = new FlowController(
flowFileEventRepo,
properties,
userService,
+ auditService,
encryptor,
/* configuredForClustering */ true,
/* NodeProtocolSender */ protocolSender);
@@ -378,6 +384,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
final FlowFileEventRepository flowFileEventRepo,
final NiFiProperties properties,
final UserService userService,
+ final AuditService auditService,
final StringEncryptor encryptor,
final boolean configuredForClustering,
final NodeProtocolSender protocolSender) {
[20/30] nifi git commit: NIFI-730: reordered states for drop flowfile
request
Posted by ma...@apache.org.
NIFI-730: reordered states for drop flowfile request
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/09a3f6da
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/09a3f6da
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/09a3f6da
Branch: refs/heads/master
Commit: 09a3f6dadd5ccff75590c53d071cc4ebe6c6175b
Parents: 98a04ee
Author: Mark Payne <ma...@hotmail.com>
Authored: Wed Oct 14 16:32:39 2015 -0400
Committer: Mark Payne <ma...@hotmail.com>
Committed: Wed Oct 14 16:32:39 2015 -0400
----------------------------------------------------------------------
.../org/apache/nifi/controller/queue/DropFlowFileState.java | 6 +++---
.../java/org/apache/nifi/controller/StandardFlowFileQueue.java | 2 +-
2 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/09a3f6da/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileState.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileState.java b/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileState.java
index 6462dfd..12dbedf 100644
--- a/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileState.java
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileState.java
@@ -23,10 +23,10 @@ package org.apache.nifi.controller.queue;
public enum DropFlowFileState {
WAITING_FOR_LOCK("Waiting for Destination Component to complete its action"),
- DROPPING_ACTIVE_FLOWFILES("Dropping FlowFiles from queue"),
- COMPLETE("Completed Successfully"),
+ DROPPING_FLOWFILES("Dropping FlowFiles from queue"),
FAILURE("Failed"),
- CANCELED("Cancelled by User");
+ CANCELED("Canceled by User"),
+ COMPLETE("Completed Successfully");
private final String description;
http://git-wip-us.apache.org/repos/asf/nifi/blob/09a3f6da/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
index b0c31e7..6e42f55 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
@@ -944,7 +944,7 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
public void run() {
writeLock.lock();
try {
- dropRequest.setState(DropFlowFileState.DROPPING_ACTIVE_FLOWFILES);
+ dropRequest.setState(DropFlowFileState.DROPPING_FLOWFILES);
logger.debug("For DropFlowFileRequest {}, original size is {}", requestIdentifier, getQueueSize());
dropRequest.setOriginalSize(getQueueSize());
[22/30] nifi git commit: NIFI-730: Updated details of Provenance
Event for when a queue is emptied
Posted by ma...@apache.org.
NIFI-730: Updated details of Provenance Event for when a queue is emptied
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/b8dbd101
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/b8dbd101
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/b8dbd101
Branch: refs/heads/master
Commit: b8dbd1018cf0b2d09bf7371849cf8b588d5f3f7c
Parents: 39a050d
Author: Mark Payne <ma...@hotmail.com>
Authored: Fri Oct 16 08:53:40 2015 -0400
Committer: Mark Payne <ma...@hotmail.com>
Committed: Fri Oct 16 08:53:40 2015 -0400
----------------------------------------------------------------------
.../java/org/apache/nifi/controller/StandardFlowFileQueue.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/b8dbd101/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
index 6e42f55..a9e80b0 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
@@ -1095,7 +1095,7 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
builder.setComponentId(getIdentifier());
builder.setComponentType("Connection");
builder.setAttributes(flowFile.getAttributes(), Collections.<String, String> emptyMap());
- builder.setDetails("Manually dropped by " + requestor);
+ builder.setDetails("FlowFile Queue emptied by " + requestor);
builder.setSourceQueueIdentifier(getIdentifier());
final ContentClaim contentClaim = flowFile.getContentClaim();
[28/30] nifi git commit: NIFI-730: - Updating logic now that original
is guaranteed to be non null. - Always reporting 100% once the drop request
has completed.
Posted by ma...@apache.org.
NIFI-730:
- Updating logic now that original is guaranteed to be non null.
- Always reporting 100% once the drop request has completed.
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/5a04021d
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/5a04021d
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/5a04021d
Branch: refs/heads/master
Commit: 5a04021dd7a61cc7c76b24405288713116bcd682
Parents: f5727cf
Author: Matt Gilman <ma...@gmail.com>
Authored: Mon Nov 2 08:59:11 2015 -0500
Committer: Matt Gilman <ma...@gmail.com>
Committed: Mon Nov 2 08:59:11 2015 -0500
----------------------------------------------------------------------
.../org/apache/nifi/web/api/dto/DtoFactory.java | 22 +++++++++++---------
1 file changed, 12 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/5a04021d/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
index d34242d..5d437cf 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
@@ -296,6 +296,10 @@ public final class DtoFactory {
return new PositionDTO(position.getX(), position.getY());
}
+ private boolean isDropRequestComplete(final DropFlowFileState state) {
+ return DropFlowFileState.COMPLETE.equals(state) || DropFlowFileState.CANCELED.equals(state) || DropFlowFileState.FAILURE.equals(state);
+ }
+
/**
* Creates a DropRequestDTO from the specified flow file status.
*
@@ -309,9 +313,7 @@ public final class DtoFactory {
dto.setLastUpdated(new Date(dropRequest.getLastUpdated()));
dto.setState(dropRequest.getState().toString());
dto.setFailureReason(dropRequest.getFailureReason());
- dto.setFinished(DropFlowFileState.COMPLETE.equals(dropRequest.getState())
- || DropFlowFileState.CANCELED.equals(dropRequest.getState())
- || DropFlowFileState.FAILURE.equals(dropRequest.getState()));
+ dto.setFinished(isDropRequestComplete(dropRequest.getState()));
final QueueSize dropped = dropRequest.getDroppedSize();
dto.setDroppedCount(dropped.getObjectCount());
@@ -323,15 +325,15 @@ public final class DtoFactory {
dto.setCurrentSize(current.getByteCount());
dto.setCurrent(FormatUtils.formatCount(current.getObjectCount()) + " / " + FormatUtils.formatDataSize(current.getByteCount()));
- if (dropRequest.getOriginalSize() != null) {
- final QueueSize original = dropRequest.getOriginalSize();
- dto.setOriginalCount(original.getObjectCount());
- dto.setOriginalSize(original.getByteCount());
- dto.setOriginal(FormatUtils.formatCount(original.getObjectCount()) + " / " + FormatUtils.formatDataSize(original.getByteCount()));
+ final QueueSize original = dropRequest.getOriginalSize();
+ dto.setOriginalCount(original.getObjectCount());
+ dto.setOriginalSize(original.getByteCount());
+ dto.setOriginal(FormatUtils.formatCount(original.getObjectCount()) + " / " + FormatUtils.formatDataSize(original.getByteCount()));
- dto.setPercentCompleted((dropped.getObjectCount() * 100) / original.getObjectCount());
+ if (isDropRequestComplete(dropRequest.getState())) {
+ dto.setPercentCompleted(100);
} else {
- dto.setPercentCompleted(0);
+ dto.setPercentCompleted((dropped.getObjectCount() * 100) / original.getObjectCount());
}
return dto;
[21/30] nifi git commit: NIFI-730: - Adding emptying a queue when
clustered.
Posted by ma...@apache.org.
NIFI-730:
- Adding emptying a queue when clustered.
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/39a050d2
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/39a050d2
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/39a050d2
Branch: refs/heads/master
Commit: 39a050d2fdc2437576d860b968ab1ec78d18fa21
Parents: 09a3f6d
Author: Matt Gilman <ma...@gmail.com>
Authored: Wed Oct 14 17:47:06 2015 -0400
Committer: Matt Gilman <ma...@gmail.com>
Committed: Wed Oct 14 17:47:06 2015 -0400
----------------------------------------------------------------------
.../controller/queue/DropFlowFileState.java | 18 ++-
.../cluster/manager/impl/WebClusterManager.java | 111 +++++++++++++++++--
.../apache/nifi/web/api/ConnectionResource.java | 26 ++++-
.../org/apache/nifi/web/api/dto/DtoFactory.java | 12 +-
4 files changed, 144 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/39a050d2/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileState.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileState.java b/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileState.java
index 12dbedf..32efcbb 100644
--- a/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileState.java
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileState.java
@@ -14,7 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.nifi.controller.queue;
/**
@@ -38,4 +37,21 @@ public enum DropFlowFileState {
public String toString() {
return description;
}
+
+ /**
+ * @param description string form of drop flow file state
+ * @return the matching DropFlowFileState or null if the description doesn't match
+ */
+ public static DropFlowFileState valueOfDescription(String description) {
+ DropFlowFileState desiredState = null;
+
+ for (DropFlowFileState state : values()) {
+ if (state.toString().equals(description)) {
+ desiredState = state;
+ break;
+ }
+ }
+
+ return desiredState;
+ }
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/39a050d2/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/impl/WebClusterManager.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/impl/WebClusterManager.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/impl/WebClusterManager.java
index bfeec7a..6b0bb64 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/impl/WebClusterManager.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/impl/WebClusterManager.java
@@ -226,14 +226,17 @@ import org.xml.sax.SAXException;
import org.xml.sax.SAXParseException;
import com.sun.jersey.api.client.ClientResponse;
+import org.apache.nifi.controller.queue.DropFlowFileState;
import org.apache.nifi.controller.service.ControllerServiceState;
import org.apache.nifi.web.api.dto.ControllerServiceDTO;
import org.apache.nifi.web.api.dto.ControllerServiceReferencingComponentDTO;
+import org.apache.nifi.web.api.dto.DropRequestDTO;
import org.apache.nifi.web.api.dto.ReportingTaskDTO;
import org.apache.nifi.web.api.entity.ControllerServiceEntity;
import org.apache.nifi.web.api.entity.ControllerServiceReferencingComponentsEntity;
import org.apache.nifi.web.api.entity.ControllerServicesEntity;
+import org.apache.nifi.web.api.entity.DropRequestEntity;
import org.apache.nifi.web.api.entity.ReportingTaskEntity;
import org.apache.nifi.web.api.entity.ReportingTasksEntity;
@@ -316,6 +319,9 @@ public class WebClusterManager implements HttpClusterManager, ProtocolHandler, C
public static final String REPORTING_TASKS_URI = "/nifi-api/controller/reporting-tasks/node";
public static final Pattern REPORTING_TASK_URI_PATTERN = Pattern.compile("/nifi-api/controller/reporting-tasks/node/[a-f0-9\\-]{36}");
+ public static final Pattern QUEUE_CONTENTS_URI = Pattern.compile("/nifi-api/controller/process-groups/(?:(?:root)|(?:[a-f0-9\\-]{36}))/connections/[a-f0-9\\-]{36}/contents");
+ public static final Pattern DROP_REQUEST_URI = Pattern.compile("/nifi-api/controller/process-groups/(?:(?:root)|(?:[a-f0-9\\-]{36}))/connections/[a-f0-9\\-]{36}/drop-requests/[a-f0-9\\-]{36}");
+
private final NiFiProperties properties;
private final HttpRequestReplicator httpRequestReplicator;
private final HttpResponseMapper httpResponseMapper;
@@ -1090,7 +1096,7 @@ public class WebClusterManager implements HttpClusterManager, ProtocolHandler, C
// Register log observer to provide bulletins when reporting task logs anything at WARN level or above
final LogRepository logRepository = LogRepositoryFactory.getRepository(id);
logRepository.addObserver(StandardProcessorNode.BULLETIN_OBSERVER_ID, LogLevel.WARN,
- new ReportingTaskLogObserver(getBulletinRepository(), taskNode));
+ new ReportingTaskLogObserver(getBulletinRepository(), taskNode));
return taskNode;
}
@@ -1385,7 +1391,7 @@ public class WebClusterManager implements HttpClusterManager, ProtocolHandler, C
// Register log observer to provide bulletins when reporting task logs anything at WARN level or above
final LogRepository logRepository = LogRepositoryFactory.getRepository(id);
logRepository.addObserver(StandardProcessorNode.BULLETIN_OBSERVER_ID, LogLevel.WARN,
- new ControllerServiceLogObserver(getBulletinRepository(), serviceNode));
+ new ControllerServiceLogObserver(getBulletinRepository(), serviceNode));
return serviceNode;
}
@@ -2465,6 +2471,16 @@ public class WebClusterManager implements HttpClusterManager, ProtocolHandler, C
return false;
}
+ private static boolean isDropRequestEndpoint(final URI uri, final String method) {
+ if ("DELETE".equalsIgnoreCase(method) && QUEUE_CONTENTS_URI.matcher(uri.getPath()).matches()) {
+ return true;
+ } else if (("GET".equalsIgnoreCase(method) || "DELETE".equalsIgnoreCase(method)) && DROP_REQUEST_URI.matcher(uri.getPath()).matches()) {
+ return true;
+ }
+
+ return false;
+ }
+
static boolean isResponseInterpreted(final URI uri, final String method) {
return isProcessorsEndpoint(uri, method) || isProcessorEndpoint(uri, method)
|| isRemoteProcessGroupsEndpoint(uri, method) || isRemoteProcessGroupEndpoint(uri, method)
@@ -2472,7 +2488,8 @@ public class WebClusterManager implements HttpClusterManager, ProtocolHandler, C
|| isTemplateEndpoint(uri, method) || isFlowSnippetEndpoint(uri, method)
|| isProvenanceQueryEndpoint(uri, method) || isProvenanceEventEndpoint(uri, method)
|| isControllerServicesEndpoint(uri, method) || isControllerServiceEndpoint(uri, method) || isControllerServiceReferenceEndpoint(uri, method)
- || isReportingTasksEndpoint(uri, method) || isReportingTaskEndpoint(uri, method);
+ || isReportingTasksEndpoint(uri, method) || isReportingTaskEndpoint(uri, method)
+ || isDropRequestEndpoint(uri, method);
}
private void mergeProcessorValidationErrors(final ProcessorDTO processor, Map<NodeIdentifier, ProcessorDTO> processorMap) {
@@ -2808,6 +2825,62 @@ public class WebClusterManager implements HttpClusterManager, ProtocolHandler, C
return normalizedValidationErrors;
}
+ /**
+ * Merges the drop requests in the specified map into the specified drop request.
+ *
+ * @param dropRequest the target drop request
+ * @param dropRequestMap the mapping of all responses being merged
+ */
+ private void mergeDropRequests(final DropRequestDTO dropRequest, final Map<NodeIdentifier, DropRequestDTO> dropRequestMap) {
+ boolean nodeWaiting = false;
+ int originalCount = 0;
+ long originalSize = 0;
+ int currentCount = 0;
+ long currentSize = 0;
+ int droppedCount = 0;
+ long droppedSize = 0;
+
+ DropFlowFileState state = null;
+ for (final Map.Entry<NodeIdentifier, DropRequestDTO> nodeEntry : dropRequestMap.entrySet()) {
+ final DropRequestDTO nodeDropRequest = nodeEntry.getValue();
+
+ currentCount += nodeDropRequest.getCurrentCount();
+ currentSize += nodeDropRequest.getCurrentSize();
+ droppedCount += nodeDropRequest.getDroppedCount();
+ droppedSize += nodeDropRequest.getDroppedSize();
+
+ if (nodeDropRequest.getOriginalCount() == null) {
+ nodeWaiting = true;
+ } else {
+ originalCount += nodeDropRequest.getOriginalCount();
+ originalSize += nodeDropRequest.getOriginalSize();
+ }
+
+ final DropFlowFileState nodeState = DropFlowFileState.valueOfDescription(nodeDropRequest.getState());
+ if (state == null || state.compareTo(nodeState) > 0) {
+ state = nodeState;
+ }
+ }
+
+ dropRequest.setCurrentCount(currentCount);
+ dropRequest.setCurrentSize(currentSize);
+ dropRequest.setCurrent(FormatUtils.formatCount(currentCount) + " / " + FormatUtils.formatDataSize(currentSize));
+
+ dropRequest.setDroppedCount(droppedCount);
+ dropRequest.setDroppedSize(droppedSize);
+ dropRequest.setDropped(FormatUtils.formatCount(droppedCount) + " / " + FormatUtils.formatDataSize(droppedSize));
+
+ if (!nodeWaiting) {
+ dropRequest.setOriginalCount(originalCount);
+ dropRequest.setOriginalSize(originalSize);
+ dropRequest.setOriginal(FormatUtils.formatCount(originalCount) + " / " + FormatUtils.formatDataSize(originalSize));
+ }
+
+ if (state != null) {
+ dropRequest.setState(state.toString());
+ }
+ }
+
// requires write lock to be already acquired unless request is not mutable
private NodeResponse mergeResponses(final URI uri, final String method, final Set<NodeResponse> nodeResponses, final boolean mutableRequest) {
// holds the one response of all the node responses to return to the client
@@ -3158,8 +3231,8 @@ public class WebClusterManager implements HttpClusterManager, ProtocolHandler, C
continue;
}
- final ControllerServiceReferencingComponentsEntity nodeResponseEntity =
- nodeResponse == clientResponse ? responseEntity : nodeResponse.getClientResponse().getEntity(ControllerServiceReferencingComponentsEntity.class);
+ final ControllerServiceReferencingComponentsEntity nodeResponseEntity
+ = nodeResponse == clientResponse ? responseEntity : nodeResponse.getClientResponse().getEntity(ControllerServiceReferencingComponentsEntity.class);
final Set<ControllerServiceReferencingComponentDTO> nodeReferencingComponents = nodeResponseEntity.getControllerServiceReferencingComponents();
resultsMap.put(nodeResponse.getNodeId(), nodeReferencingComponents);
@@ -3218,6 +3291,24 @@ public class WebClusterManager implements HttpClusterManager, ProtocolHandler, C
// create a new client response
clientResponse = new NodeResponse(clientResponse, responseEntity);
+ } else if (hasSuccessfulClientResponse && isDropRequestEndpoint(uri, method)) {
+ final DropRequestEntity responseEntity = clientResponse.getClientResponse().getEntity(DropRequestEntity.class);
+ final DropRequestDTO dropRequest = responseEntity.getDropRequest();
+
+ final Map<NodeIdentifier, DropRequestDTO> resultsMap = new HashMap<>();
+ for (final NodeResponse nodeResponse : updatedNodesMap.values()) {
+ if (problematicNodeResponses.contains(nodeResponse)) {
+ continue;
+ }
+
+ final DropRequestEntity nodeResponseEntity = nodeResponse == clientResponse ? responseEntity : nodeResponse.getClientResponse().getEntity(DropRequestEntity.class);
+ final DropRequestDTO nodeDropRequest = nodeResponseEntity.getDropRequest();
+
+ resultsMap.put(nodeResponse.getNodeId(), nodeDropRequest);
+ }
+ mergeDropRequests(dropRequest, resultsMap);
+
+ clientResponse = new NodeResponse(clientResponse, responseEntity);
} else {
if (!nodeResponsesToDrain.isEmpty()) {
drainResponses(nodeResponsesToDrain);
@@ -3270,12 +3361,12 @@ public class WebClusterManager implements HttpClusterManager, ProtocolHandler, C
}
/**
- * Determines if all problematic responses were due to 404 NOT_FOUND. Assumes that problematicNodeResponses is not empty and
- * is not comprised of responses from all nodes in the cluster (at least one node contained the counter in question).
+ * Determines if all problematic responses were due to 404 NOT_FOUND. Assumes that problematicNodeResponses is not empty and is not comprised of responses from all nodes in the cluster (at least
+ * one node contained the counter in question).
*
- * @param problematicNodeResponses The problematic node responses
- * @param uri The URI for the request
- * @return Whether all problematic node responses were due to a missing counter
+ * @param problematicNodeResponses The problematic node responses
+ * @param uri The URI for the request
+ * @return Whether all problematic node responses were due to a missing counter
*/
private boolean isMissingCounter(final Set<NodeResponse> problematicNodeResponses, final URI uri) {
if (isCountersEndpoint(uri)) {
http://git-wip-us.apache.org/repos/asf/nifi/blob/39a050d2/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectionResource.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectionResource.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectionResource.java
index b170d39..6741348 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectionResource.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectionResource.java
@@ -894,6 +894,7 @@ public class ConnectionResource extends ApplicationResource {
/**
* Drops the flowfiles in the queue of the specified connection.
*
+ * @param httpServletRequest request
* @param clientId Optional client id. If the client id is not specified, a new one will be generated. This value (whether specified or generated) is included in the response.
* @param id The id of the connection
* @return A dropRequestEntity
@@ -920,6 +921,7 @@ public class ConnectionResource extends ApplicationResource {
}
)
public Response dropQueueContents(
+ @Context HttpServletRequest httpServletRequest,
@ApiParam(
value = "If the client id is not specified, new one will be generated. This value (whether specified or generated) is included in the response.",
required = false
@@ -936,6 +938,12 @@ public class ConnectionResource extends ApplicationResource {
return clusterManager.applyRequest(HttpMethod.DELETE, getAbsolutePath(), getRequestParameters(true), getHeaders()).getResponse();
}
+ // handle expects request (usually from the cluster manager)
+ final String expects = httpServletRequest.getHeader(WebClusterManager.NCM_EXPECTS_HTTP_HEADER);
+ if (expects != null) {
+ return generateContinueResponse().build();
+ }
+
// ensure the id is the same across the cluster
final String dropRequestId;
final ClusterContext clusterContext = ClusterContextThreadLocal.getContext();
@@ -947,7 +955,7 @@ public class ConnectionResource extends ApplicationResource {
// submit the drop request
final DropRequestDTO dropRequest = serviceFacade.createFlowFileDropRequest(groupId, id, dropRequestId);
- dropRequest.setUri(generateResourceUri("controller", "process-groups", groupId, "connections", id, "contents", "drop-requests", dropRequest.getId()));
+ dropRequest.setUri(generateResourceUri("controller", "process-groups", groupId, "connections", id, "drop-requests", dropRequest.getId()));
// create the revision
final RevisionDTO revision = new RevisionDTO();
@@ -978,7 +986,7 @@ public class ConnectionResource extends ApplicationResource {
@GET
@Consumes(MediaType.WILDCARD)
@Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML})
- @Path("/{connection-id}/contents/drop-requests/{drop-request-id}")
+ @Path("/{connection-id}/drop-requests/{drop-request-id}")
@PreAuthorize("hasRole('ROLE_DFM')")
@ApiOperation(
value = "Gets the current status of a drop request for the specified connection.",
@@ -1020,7 +1028,7 @@ public class ConnectionResource extends ApplicationResource {
// get the drop request
final DropRequestDTO dropRequest = serviceFacade.getFlowFileDropRequest(groupId, connectionId, dropRequestId);
- dropRequest.setUri(generateResourceUri("controller", "process-groups", groupId, "connections", connectionId, "contents", "drop-requests", dropRequestId));
+ dropRequest.setUri(generateResourceUri("controller", "process-groups", groupId, "connections", connectionId, "drop-requests", dropRequestId));
// create the revision
final RevisionDTO revision = new RevisionDTO();
@@ -1037,6 +1045,7 @@ public class ConnectionResource extends ApplicationResource {
/**
* Deletes the specified drop request.
*
+ * @param httpServletRequest request
* @param clientId Optional client id. If the client id is not specified, a new one will be generated. This value (whether specified or generated) is included in the response.
* @param connectionId The connection id
* @param dropRequestId The drop request id
@@ -1045,7 +1054,7 @@ public class ConnectionResource extends ApplicationResource {
@DELETE
@Consumes(MediaType.WILDCARD)
@Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML})
- @Path("/{connection-id}/contents/drop-requests/{drop-request-id}")
+ @Path("/{connection-id}/drop-requests/{drop-request-id}")
@PreAuthorize("hasRole('ROLE_DFM')")
@ApiOperation(
value = "Cancels and/or removes a request drop of the contents in this connection.",
@@ -1064,6 +1073,7 @@ public class ConnectionResource extends ApplicationResource {
}
)
public Response removeDropRequest(
+ @Context HttpServletRequest httpServletRequest,
@ApiParam(
value = "If the client id is not specified, new one will be generated. This value (whether specified or generated) is included in the response.",
required = false
@@ -1085,9 +1095,15 @@ public class ConnectionResource extends ApplicationResource {
return clusterManager.applyRequest(HttpMethod.DELETE, getAbsolutePath(), getRequestParameters(true), getHeaders()).getResponse();
}
+ // handle expects request (usually from the cluster manager)
+ final String expects = httpServletRequest.getHeader(WebClusterManager.NCM_EXPECTS_HTTP_HEADER);
+ if (expects != null) {
+ return generateContinueResponse().build();
+ }
+
// delete the drop request
final DropRequestDTO dropRequest = serviceFacade.deleteFlowFileDropRequest(groupId, connectionId, dropRequestId);
- dropRequest.setUri(generateResourceUri("controller", "process-groups", groupId, "connections", connectionId, "contents", "drop-requests", dropRequestId));
+ dropRequest.setUri(generateResourceUri("controller", "process-groups", groupId, "connections", connectionId, "drop-requests", dropRequestId));
// create the revision
final RevisionDTO revision = new RevisionDTO();
http://git-wip-us.apache.org/repos/asf/nifi/blob/39a050d2/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
index 264268b..0758ce2 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
@@ -315,6 +315,11 @@ public final class DtoFactory {
dto.setDroppedSize(dropped.getByteCount());
dto.setDropped(FormatUtils.formatCount(dropped.getObjectCount()) + " / " + FormatUtils.formatDataSize(dropped.getByteCount()));
+ final QueueSize current = dropRequest.getCurrentSize();
+ dto.setCurrentCount(current.getObjectCount());
+ dto.setCurrentSize(current.getByteCount());
+ dto.setCurrent(FormatUtils.formatCount(current.getObjectCount()) + " / " + FormatUtils.formatDataSize(current.getByteCount()));
+
if (dropRequest.getOriginalSize() != null) {
final QueueSize original = dropRequest.getOriginalSize();
dto.setOriginalCount(original.getObjectCount());
@@ -326,13 +331,6 @@ public final class DtoFactory {
dto.setPercentCompleted(0);
}
- if (dropRequest.getCurrentSize() != null) {
- final QueueSize current = dropRequest.getCurrentSize();
- dto.setCurrentCount(current.getObjectCount());
- dto.setCurrentSize(current.getByteCount());
- dto.setCurrent(FormatUtils.formatCount(current.getObjectCount()) + " / " + FormatUtils.formatDataSize(current.getByteCount()));
- }
-
return dto;
}
[04/30] nifi git commit: NIFI-730: Implemented swapping in and out
on-demand by the FlowFileQueue rather than in a background thread
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/49a781df/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/test/java/org/apache/nifi/controller/TestStandardFlowFileQueue.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/test/java/org/apache/nifi/controller/TestStandardFlowFileQueue.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/test/java/org/apache/nifi/controller/TestStandardFlowFileQueue.java
new file mode 100644
index 0000000..66f32d8
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/test/java/org/apache/nifi/controller/TestStandardFlowFileQueue.java
@@ -0,0 +1,330 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.controller;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.nifi.connectable.Connectable;
+import org.apache.nifi.connectable.Connection;
+import org.apache.nifi.controller.queue.FlowFileQueue;
+import org.apache.nifi.controller.queue.QueueSize;
+import org.apache.nifi.controller.repository.FlowFileRecord;
+import org.apache.nifi.controller.repository.FlowFileRepository;
+import org.apache.nifi.controller.repository.FlowFileSwapManager;
+import org.apache.nifi.controller.repository.SwapManagerInitializationContext;
+import org.apache.nifi.controller.repository.claim.ContentClaim;
+import org.apache.nifi.controller.repository.claim.ResourceClaimManager;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.FlowFilePrioritizer;
+import org.apache.nifi.provenance.ProvenanceEventRepository;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+public class TestStandardFlowFileQueue {
+ private TestSwapManager swapManager = null;
+ private StandardFlowFileQueue queue = null;
+
+ @Before
+ public void setup() {
+ final Connection connection = Mockito.mock(Connection.class);
+ Mockito.when(connection.getSource()).thenReturn(Mockito.mock(Connectable.class));
+ Mockito.when(connection.getDestination()).thenReturn(Mockito.mock(Connectable.class));
+
+ final ProcessScheduler scheduler = Mockito.mock(ProcessScheduler.class);
+ swapManager = new TestSwapManager();
+
+ final FlowFileRepository flowFileRepo = Mockito.mock(FlowFileRepository.class);
+ final ProvenanceEventRepository provRepo = Mockito.mock(ProvenanceEventRepository.class);
+ final ResourceClaimManager claimManager = Mockito.mock(ResourceClaimManager.class);
+
+ queue = new StandardFlowFileQueue("id", connection, flowFileRepo, provRepo, claimManager, scheduler, swapManager, null, 10000);
+ TestFlowFile.idGenerator.set(0L);
+ }
+
+
+ @Test
+ public void testSwapOutOccurs() {
+ for (int i = 0; i < 10000; i++) {
+ queue.put(new TestFlowFile());
+ assertEquals(0, swapManager.swapOutCalledCount);
+ assertEquals(i + 1, queue.size().getObjectCount());
+ assertEquals(i + 1, queue.size().getByteCount());
+ }
+
+ for (int i = 0; i < 9999; i++) {
+ queue.put(new TestFlowFile());
+ assertEquals(0, swapManager.swapOutCalledCount);
+ assertEquals(i + 10001, queue.size().getObjectCount());
+ assertEquals(i + 10001, queue.size().getByteCount());
+ }
+
+ queue.put(new TestFlowFile(1000));
+ assertEquals(1, swapManager.swapOutCalledCount);
+ assertEquals(20000, queue.size().getObjectCount());
+ assertEquals(20999, queue.size().getByteCount());
+
+ assertEquals(10000, queue.getActiveQueueSize().getObjectCount());
+ }
+
+ @Test
+ public void testLowestPrioritySwappedOutFirst() {
+ final List<FlowFilePrioritizer> prioritizers = new ArrayList<>();
+ prioritizers.add(new FlowFileSizePrioritizer());
+ queue.setPriorities(prioritizers);
+
+ long maxSize = 20000;
+ for (int i = 1; i <= 20000; i++) {
+ queue.put(new TestFlowFile(maxSize - i));
+ }
+
+ assertEquals(1, swapManager.swapOutCalledCount);
+ assertEquals(20000, queue.size().getObjectCount());
+
+ assertEquals(10000, queue.getActiveQueueSize().getObjectCount());
+ final List<FlowFileRecord> flowFiles = queue.poll(Integer.MAX_VALUE, new HashSet<FlowFileRecord>());
+ assertEquals(10000, flowFiles.size());
+ for (int i = 0; i < 10000; i++) {
+ assertEquals(i, flowFiles.get(i).getSize());
+ }
+ }
+
+ @Test
+ public void testSwapIn() {
+ for (int i = 1; i <= 20000; i++) {
+ queue.put(new TestFlowFile());
+ }
+
+ assertEquals(1, swapManager.swappedOut.size());
+ queue.put(new TestFlowFile());
+ assertEquals(1, swapManager.swappedOut.size());
+
+ final Set<FlowFileRecord> exp = new HashSet<>();
+ for (int i = 0; i < 9999; i++) {
+ assertNotNull(queue.poll(exp));
+ }
+
+ assertEquals(0, swapManager.swapInCalledCount);
+ assertEquals(1, queue.getActiveQueueSize().getObjectCount());
+ assertNotNull(queue.poll(exp));
+
+ assertEquals(0, swapManager.swapInCalledCount);
+ assertEquals(0, queue.getActiveQueueSize().getObjectCount());
+
+ assertEquals(1, swapManager.swapOutCalledCount);
+
+ assertNotNull(queue.poll(exp)); // this should trigger a swap-in of 10,000 records, and then pull 1 off the top.
+ assertEquals(1, swapManager.swapInCalledCount);
+ assertEquals(9999, queue.getActiveQueueSize().getObjectCount());
+
+ assertTrue(swapManager.swappedOut.isEmpty());
+
+ queue.poll(exp);
+
+ }
+
+
+ private class TestSwapManager implements FlowFileSwapManager {
+ private final Map<String, List<FlowFileRecord>> swappedOut = new HashMap<>();
+ int swapOutCalledCount = 0;
+ int swapInCalledCount = 0;
+
+
+ @Override
+ public void initialize(final SwapManagerInitializationContext initializationContext) {
+
+ }
+
+ @Override
+ public String swapOut(List<FlowFileRecord> flowFiles, FlowFileQueue flowFileQueue) throws IOException {
+ swapOutCalledCount++;
+ final String location = UUID.randomUUID().toString();
+ swappedOut.put(location, new ArrayList<FlowFileRecord>(flowFiles));
+ return location;
+ }
+
+ @Override
+ public List<FlowFileRecord> peek(String swapLocation, final FlowFileQueue flowFileQueue) throws IOException {
+ return new ArrayList<FlowFileRecord>(swappedOut.get(swapLocation));
+ }
+
+ @Override
+ public List<FlowFileRecord> swapIn(String swapLocation, FlowFileQueue flowFileQueue) throws IOException {
+ swapInCalledCount++;
+ return swappedOut.remove(swapLocation);
+ }
+
+ @Override
+ public List<String> recoverSwapLocations(FlowFileQueue flowFileQueue) throws IOException {
+ return new ArrayList<String>(swappedOut.keySet());
+ }
+
+ @Override
+ public void dropSwappedFlowFiles(String swapLocation, final FlowFileQueue flowFileQueue, String user) {
+
+ }
+
+ @Override
+ public QueueSize getSwapSize(String swapLocation) throws IOException {
+ final List<FlowFileRecord> flowFiles = swappedOut.get(swapLocation);
+ if (flowFiles == null) {
+ return new QueueSize(0, 0L);
+ }
+
+ int count = 0;
+ long size = 0L;
+ for (final FlowFileRecord flowFile : flowFiles) {
+ count++;
+ size += flowFile.getSize();
+ }
+
+ return new QueueSize(count, size);
+ }
+
+ @Override
+ public Long getMaxRecordId(String swapLocation) throws IOException {
+ final List<FlowFileRecord> flowFiles = swappedOut.get(swapLocation);
+ if (flowFiles == null) {
+ return null;
+ }
+
+ Long max = null;
+ for (final FlowFileRecord flowFile : flowFiles) {
+ if (max == null || flowFile.getId() > max) {
+ max = flowFile.getId();
+ }
+ }
+
+ return max;
+ }
+
+ @Override
+ public void purge() {
+ swappedOut.clear();
+ }
+ }
+
+
+ private static class TestFlowFile implements FlowFileRecord {
+ private static final AtomicLong idGenerator = new AtomicLong(0L);
+
+ private final long id = idGenerator.getAndIncrement();
+ private final long entryDate = System.currentTimeMillis();
+ private final Map<String, String> attributes;
+ private final long size;
+
+ public TestFlowFile() {
+ this(1L);
+ }
+
+ public TestFlowFile(final long size) {
+ this(new HashMap<String, String>(), size);
+ }
+
+ public TestFlowFile(final Map<String, String> attributes, final long size) {
+ this.attributes = attributes;
+ this.size = size;
+ }
+
+
+ @Override
+ public long getId() {
+ return id;
+ }
+
+ @Override
+ public long getEntryDate() {
+ return entryDate;
+ }
+
+ @Override
+ public long getLineageStartDate() {
+ return entryDate;
+ }
+
+ @Override
+ public Long getLastQueueDate() {
+ return null;
+ }
+
+ @Override
+ public Set<String> getLineageIdentifiers() {
+ return Collections.emptySet();
+ }
+
+ @Override
+ public boolean isPenalized() {
+ return false;
+ }
+
+ @Override
+ public String getAttribute(String key) {
+ return attributes.get(key);
+ }
+
+ @Override
+ public long getSize() {
+ return size;
+ }
+
+ @Override
+ public Map<String, String> getAttributes() {
+ return Collections.unmodifiableMap(attributes);
+ }
+
+ @Override
+ public int compareTo(final FlowFile o) {
+ return Long.compare(id, o.getId());
+ }
+
+ @Override
+ public long getPenaltyExpirationMillis() {
+ return 0;
+ }
+
+ @Override
+ public ContentClaim getContentClaim() {
+ return null;
+ }
+
+ @Override
+ public long getContentClaimOffset() {
+ return 0;
+ }
+ }
+
+ private static class FlowFileSizePrioritizer implements FlowFilePrioritizer {
+ @Override
+ public int compare(final FlowFile o1, final FlowFile o2) {
+ return Long.compare(o1.getSize(), o2.getSize());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/49a781df/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/connectable/StandardConnection.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/connectable/StandardConnection.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/connectable/StandardConnection.java
index f0a6d8a..d43a3db 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/connectable/StandardConnection.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/connectable/StandardConnection.java
@@ -32,11 +32,14 @@ import org.apache.nifi.controller.ProcessScheduler;
import org.apache.nifi.controller.StandardFlowFileQueue;
import org.apache.nifi.controller.queue.FlowFileQueue;
import org.apache.nifi.controller.repository.FlowFileRecord;
+import org.apache.nifi.controller.repository.FlowFileRepository;
import org.apache.nifi.controller.repository.FlowFileSwapManager;
+import org.apache.nifi.controller.repository.claim.ResourceClaimManager;
import org.apache.nifi.events.EventReporter;
import org.apache.nifi.groups.ProcessGroup;
import org.apache.nifi.processor.FlowFileFilter;
import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.provenance.ProvenanceEventRepository;
import org.apache.nifi.util.NiFiProperties;
/**
@@ -66,7 +69,8 @@ public final class StandardConnection implements Connection {
destination = new AtomicReference<>(builder.destination);
relationships = new AtomicReference<>(Collections.unmodifiableCollection(builder.relationships));
scheduler = builder.scheduler;
- flowFileQueue = new StandardFlowFileQueue(id, this, scheduler, builder.swapManager, builder.eventReporter, NiFiProperties.getInstance().getQueueSwapThreshold());
+ flowFileQueue = new StandardFlowFileQueue(id, this, builder.flowFileRepository, builder.provenanceRepository, builder.resourceClaimManager,
+ scheduler, builder.swapManager, builder.eventReporter, NiFiProperties.getInstance().getQueueSwapThreshold());
hashCode = new HashCodeBuilder(7, 67).append(id).toHashCode();
}
@@ -262,6 +266,9 @@ public final class StandardConnection implements Connection {
private Collection<Relationship> relationships;
private FlowFileSwapManager swapManager;
private EventReporter eventReporter;
+ private FlowFileRepository flowFileRepository;
+ private ProvenanceEventRepository provenanceRepository;
+ private ResourceClaimManager resourceClaimManager;
public Builder(final ProcessScheduler scheduler) {
this.scheduler = scheduler;
@@ -318,6 +325,21 @@ public final class StandardConnection implements Connection {
return this;
}
+ public Builder flowFileRepository(final FlowFileRepository flowFileRepository) {
+ this.flowFileRepository = flowFileRepository;
+ return this;
+ }
+
+ public Builder provenanceRepository(final ProvenanceEventRepository provenanceRepository) {
+ this.provenanceRepository = provenanceRepository;
+ return this;
+ }
+
+ public Builder resourceClaimManager(final ResourceClaimManager resourceClaimManager) {
+ this.resourceClaimManager = resourceClaimManager;
+ return this;
+ }
+
public StandardConnection build() {
if (source == null) {
throw new IllegalStateException("Cannot build a Connection without a Source");
@@ -328,6 +350,15 @@ public final class StandardConnection implements Connection {
if (swapManager == null) {
throw new IllegalStateException("Cannot build a Connection without a FlowFileSwapManager");
}
+ if (flowFileRepository == null) {
+ throw new IllegalStateException("Cannot build a Connection without a FlowFile Repository");
+ }
+ if (provenanceRepository == null) {
+ throw new IllegalStateException("Cannot build a Connection without a Provenance Repository");
+ }
+ if (resourceClaimManager == null) {
+ throw new IllegalStateException("Cannot build a Connection without a Resource Claim Manager");
+ }
if (relationships == null) {
relationships = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/nifi/blob/49a781df/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FileSystemSwapManager.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FileSystemSwapManager.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FileSystemSwapManager.java
index 7ab56ed..c4a86f2 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FileSystemSwapManager.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FileSystemSwapManager.java
@@ -28,6 +28,7 @@ import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
@@ -79,7 +80,6 @@ public class FileSystemSwapManager implements FlowFileSwapManager {
private EventReporter eventReporter;
private ResourceClaimManager claimManager;
-
public FileSystemSwapManager() {
final NiFiProperties properties = NiFiProperties.getInstance();
final Path flowFileRepoPath = properties.getFlowFileRepositoryPath();
@@ -111,6 +111,10 @@ public class FileSystemSwapManager implements FlowFileSwapManager {
try (final FileOutputStream fos = new FileOutputStream(swapTempFile)) {
serializeFlowFiles(toSwap, flowFileQueue, swapLocation, fos);
fos.getFD().sync();
+ } catch (final IOException ioe) {
+ // we failed to write out the entire swap file. Delete the temporary file, if we can.
+ swapTempFile.delete();
+ throw ioe;
}
if (swapTempFile.renameTo(swapFile)) {
@@ -133,25 +137,6 @@ public class FileSystemSwapManager implements FlowFileSwapManager {
warn("Swapped in FlowFiles from file " + swapFile.getAbsolutePath() + " but failed to delete the file; this file should be cleaned up manually");
}
- // TODO: When FlowFile Queue performs this operation, it needs to take the following error handling logic into account:
-
- /*
- * } catch (final EOFException eof) {
- * error("Failed to Swap In FlowFiles for " + flowFileQueue + " due to: Corrupt Swap File; will remove this Swap File: " + swapFile);
- *
- * if (!swapFile.delete()) {
- * warn("Failed to remove corrupt Swap File " + swapFile + "; This file should be cleaned up manually");
- * }
- * } catch (final FileNotFoundException fnfe) {
- * error("Failed to Swap In FlowFiles for " + flowFileQueue + " due to: Could not find Swap File " + swapFile);
- * } catch (final Exception e) {
- * error("Failed to Swap In FlowFiles for " + flowFileQueue + " due to " + e, e);
- *
- * if (swapFile != null) {
- * queue.add(swapFile);
- * }
- * }
- */
return swappedFlowFiles;
}
@@ -165,7 +150,7 @@ public class FileSystemSwapManager implements FlowFileSwapManager {
final List<FlowFileRecord> swappedFlowFiles;
try (final InputStream fis = new FileInputStream(swapFile);
final DataInputStream in = new DataInputStream(fis)) {
- swappedFlowFiles = deserializeFlowFiles(in, flowFileQueue, swapLocation, claimManager);
+ swappedFlowFiles = deserializeFlowFiles(in, swapLocation, flowFileQueue, claimManager);
}
return swappedFlowFiles;
@@ -189,6 +174,12 @@ public class FileSystemSwapManager implements FlowFileSwapManager {
}
@Override
+ public void dropSwappedFlowFiles(final String swapLocation, final FlowFileQueue flowFileQueue, final String user) throws IOException {
+
+ }
+
+
+ @Override
public List<String> recoverSwapLocations(final FlowFileQueue flowFileQueue) throws IOException {
final File[] swapFiles = storageDirectory.listFiles(new FilenameFilter() {
@Override
@@ -322,7 +313,7 @@ public class FileSystemSwapManager implements FlowFileSwapManager {
}
- public int serializeFlowFiles(final List<FlowFileRecord> toSwap, final FlowFileQueue queue, final String swapLocation, final OutputStream destination) throws IOException {
+ public static int serializeFlowFiles(final List<FlowFileRecord> toSwap, final FlowFileQueue queue, final String swapLocation, final OutputStream destination) throws IOException {
if (toSwap == null || toSwap.isEmpty()) {
return 0;
}
@@ -396,8 +387,8 @@ public class FileSystemSwapManager implements FlowFileSwapManager {
return toSwap.size();
}
- private void writeString(final String toWrite, final OutputStream out) throws IOException {
- final byte[] bytes = toWrite.getBytes("UTF-8");
+ private static void writeString(final String toWrite, final OutputStream out) throws IOException {
+ final byte[] bytes = toWrite.getBytes(StandardCharsets.UTF_8);
final int utflen = bytes.length;
if (utflen < 65535) {
@@ -415,26 +406,29 @@ public class FileSystemSwapManager implements FlowFileSwapManager {
}
}
- static List<FlowFileRecord> deserializeFlowFiles(final DataInputStream in, final FlowFileQueue queue, final String swapLocation, final ResourceClaimManager claimManager) throws IOException {
+ static List<FlowFileRecord> deserializeFlowFiles(final DataInputStream in, final String swapLocation, final FlowFileQueue queue, final ResourceClaimManager claimManager) throws IOException {
final int swapEncodingVersion = in.readInt();
if (swapEncodingVersion > SWAP_ENCODING_VERSION) {
throw new IOException("Cannot swap FlowFiles in from SwapFile because the encoding version is "
+ swapEncodingVersion + ", which is too new (expecting " + SWAP_ENCODING_VERSION + " or less)");
}
- final String connectionId = in.readUTF();
+ final String connectionId = in.readUTF(); // Connection ID
if (!connectionId.equals(queue.getIdentifier())) {
- throw new IllegalArgumentException("Cannot restore contents from FlowFile Swap File " + swapLocation +
- " because the file indicates that records belong to Connection with ID " + connectionId + " but attempted to swap those records into " + queue);
+ throw new IllegalArgumentException("Cannot deserialize FlowFiles from Swap File at location " + swapLocation +
+ " because those FlowFiles belong to Connection with ID " + connectionId + " and an attempt was made to swap them into a Connection with ID " + queue.getIdentifier());
}
final int numRecords = in.readInt();
in.readLong(); // Content Size
+ if (swapEncodingVersion > 7) {
+ in.readLong(); // Max Record ID
+ }
return deserializeFlowFiles(in, numRecords, swapEncodingVersion, false, claimManager);
}
- static List<FlowFileRecord> deserializeFlowFiles(final DataInputStream in, final int numFlowFiles,
+ private static List<FlowFileRecord> deserializeFlowFiles(final DataInputStream in, final int numFlowFiles,
final int serializationVersion, final boolean incrementContentClaims, final ResourceClaimManager claimManager) throws IOException {
final List<FlowFileRecord> flowFiles = new ArrayList<>();
for (int i = 0; i < numFlowFiles; i++) {
@@ -543,7 +537,7 @@ public class FileSystemSwapManager implements FlowFileSwapManager {
}
final byte[] bytes = new byte[numBytes];
fillBuffer(in, bytes, numBytes);
- return new String(bytes, "UTF-8");
+ return new String(bytes, StandardCharsets.UTF_8);
}
private static Integer readFieldLength(final InputStream in) throws IOException {
http://git-wip-us.apache.org/repos/asf/nifi/blob/49a781df/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java
index 23746ce..20f2642 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java
@@ -286,7 +286,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
private final NodeProtocolSender protocolSender;
private final ScheduledExecutorService clusterTaskExecutor = new FlowEngine(3, "Clustering Tasks");
- private final ResourceClaimManager contentClaimManager = new StandardResourceClaimManager();
+ private final ResourceClaimManager resourceClaimManager = new StandardResourceClaimManager();
// guarded by rwLock
/**
@@ -393,7 +393,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
timerDrivenEngineRef = new AtomicReference<>(new FlowEngine(maxTimerDrivenThreads.get(), "Timer-Driven Process"));
eventDrivenEngineRef = new AtomicReference<>(new FlowEngine(maxEventDrivenThreads.get(), "Event-Driven Process"));
- final FlowFileRepository flowFileRepo = createFlowFileRepository(properties, contentClaimManager);
+ final FlowFileRepository flowFileRepo = createFlowFileRepository(properties, resourceClaimManager);
flowFileRepository = flowFileRepo;
flowFileEventRepository = flowFileEventRepo;
counterRepositoryRef = new AtomicReference<CounterRepository>(new StandardCounterRepository());
@@ -668,7 +668,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
try {
final ContentRepository contentRepo = NarThreadContextClassLoader.createInstance(implementationClassName, ContentRepository.class);
synchronized (contentRepo) {
- contentRepo.initialize(contentClaimManager);
+ contentRepo.initialize(resourceClaimManager);
}
return contentRepo;
} catch (final Exception e) {
@@ -728,11 +728,12 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
// Create and initialize a FlowFileSwapManager for this connection
final FlowFileSwapManager swapManager = createSwapManager(properties);
final EventReporter eventReporter = createEventReporter(getBulletinRepository());
+
try (final NarCloseable narCloseable = NarCloseable.withNarLoader()) {
final SwapManagerInitializationContext initializationContext = new SwapManagerInitializationContext() {
@Override
public ResourceClaimManager getResourceClaimManager() {
- return getResourceClaimManager();
+ return resourceClaimManager;
}
@Override
@@ -756,6 +757,9 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
.destination(destination)
.swapManager(swapManager)
.eventReporter(eventReporter)
+ .resourceClaimManager(resourceClaimManager)
+ .flowFileRepository(flowFileRepository)
+ .provenanceRepository(provenanceEventRepository)
.build();
}
@@ -3188,7 +3192,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
throw new IllegalArgumentException("Input Content Claim not specified");
}
- final ResourceClaim resourceClaim = contentClaimManager.newResourceClaim(provEvent.getPreviousContentClaimContainer(), provEvent.getPreviousContentClaimSection(),
+ final ResourceClaim resourceClaim = resourceClaimManager.newResourceClaim(provEvent.getPreviousContentClaimContainer(), provEvent.getPreviousContentClaimSection(),
provEvent.getPreviousContentClaimIdentifier(), false);
claim = new StandardContentClaim(resourceClaim, provEvent.getPreviousContentClaimOffset());
offset = provEvent.getPreviousContentClaimOffset() == null ? 0L : provEvent.getPreviousContentClaimOffset();
@@ -3198,7 +3202,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
throw new IllegalArgumentException("Output Content Claim not specified");
}
- final ResourceClaim resourceClaim = contentClaimManager.newResourceClaim(provEvent.getContentClaimContainer(), provEvent.getContentClaimSection(),
+ final ResourceClaim resourceClaim = resourceClaimManager.newResourceClaim(provEvent.getContentClaimContainer(), provEvent.getContentClaimSection(),
provEvent.getContentClaimIdentifier(), false);
claim = new StandardContentClaim(resourceClaim, provEvent.getContentClaimOffset());
@@ -3247,7 +3251,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
}
try {
- final ResourceClaim resourceClaim = contentClaimManager.newResourceClaim(contentClaimContainer, contentClaimSection, contentClaimId, false);
+ final ResourceClaim resourceClaim = resourceClaimManager.newResourceClaim(contentClaimContainer, contentClaimSection, contentClaimId, false);
final ContentClaim contentClaim = new StandardContentClaim(resourceClaim, event.getPreviousContentClaimOffset());
if (!contentRepository.isAccessible(contentClaim)) {
@@ -3327,17 +3331,17 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R
}
// Create the ContentClaim
- final ResourceClaim resourceClaim = contentClaimManager.newResourceClaim(event.getPreviousContentClaimContainer(),
+ final ResourceClaim resourceClaim = resourceClaimManager.newResourceClaim(event.getPreviousContentClaimContainer(),
event.getPreviousContentClaimSection(), event.getPreviousContentClaimIdentifier(), false);
// Increment Claimant Count, since we will now be referencing the Content Claim
- contentClaimManager.incrementClaimantCount(resourceClaim);
+ resourceClaimManager.incrementClaimantCount(resourceClaim);
final long claimOffset = event.getPreviousContentClaimOffset() == null ? 0L : event.getPreviousContentClaimOffset().longValue();
final StandardContentClaim contentClaim = new StandardContentClaim(resourceClaim, claimOffset);
contentClaim.setLength(event.getPreviousFileSize() == null ? -1L : event.getPreviousFileSize());
if (!contentRepository.isAccessible(contentClaim)) {
- contentClaimManager.decrementClaimantCount(resourceClaim);
+ resourceClaimManager.decrementClaimantCount(resourceClaim);
throw new IllegalStateException("Cannot replay data from Provenance Event because the data is no longer available in the Content Repository");
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/49a781df/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
index a32a485..cfbb770 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
@@ -81,9 +81,11 @@ import org.slf4j.LoggerFactory;
* <p>
* Provides a ProcessSession that ensures all accesses, changes and transfers
* occur in an atomic manner for all FlowFiles including their contents and
- * attributes</p>
+ * attributes
+ * </p>
* <p>
- * NOT THREAD SAFE</p>
+ * NOT THREAD SAFE
+ * </p>
* <p/>
*/
public final class StandardProcessSession implements ProcessSession, ProvenanceEventEnricher {
@@ -104,7 +106,7 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
private final Map<String, Long> globalCounters = new HashMap<>();
private final Map<ContentClaim, ByteCountingOutputStream> appendableStreams = new HashMap<>();
private final ProcessContext context;
- private final Set<FlowFile> recursionSet = new HashSet<>();//set used to track what is currently being operated on to prevent logic failures if recursive calls occurring
+ private final Set<FlowFile> recursionSet = new HashSet<>();// set used to track what is currently being operated on to prevent logic failures if recursive calls occurring
private final Set<Path> deleteOnCommit = new HashSet<>();
private final long sessionId;
private final String connectableDescription;
@@ -114,7 +116,7 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
private final StandardProvenanceReporter provenanceReporter;
- private int removedCount = 0; // number of flowfiles removed in this session
+ private int removedCount = 0; // number of flowfiles removed in this session
private long removedBytes = 0L; // size of all flowfiles removed in this session
private final LongHolder bytesRead = new LongHolder(0L);
private final LongHolder bytesWritten = new LongHolder(0L);
@@ -169,7 +171,7 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
}
this.provenanceReporter = new StandardProvenanceReporter(this, connectable.getIdentifier(), componentType,
- context.getProvenanceRepository(), this);
+ context.getProvenanceRepository(), this);
this.sessionId = idGenerator.getAndIncrement();
this.connectableDescription = description;
@@ -196,7 +198,7 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
// Processor-reported events.
List<ProvenanceEventRecord> autoTerminatedEvents = null;
- //validate that all records have a transfer relationship for them and if so determine the destination node and clone as necessary
+ // validate that all records have a transfer relationship for them and if so determine the destination node and clone as necessary
final Map<FlowFileRecord, StandardRepositoryRecord> toAdd = new HashMap<>();
for (final StandardRepositoryRecord record : records.values()) {
if (record.isMarkedForDelete()) {
@@ -235,11 +237,11 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
}
}
} else {
- final Connection finalDestination = destinations.remove(destinations.size() - 1); //remove last element
+ final Connection finalDestination = destinations.remove(destinations.size() - 1); // remove last element
record.setDestination(finalDestination.getFlowFileQueue());
incrementConnectionInputCounts(finalDestination, record);
- for (final Connection destination : destinations) { //iterate over remaining destinations and "clone" as needed
+ for (final Connection destination : destinations) { // iterate over remaining destinations and "clone" as needed
incrementConnectionInputCounts(destination, record);
final FlowFileRecord currRec = record.getCurrent();
final StandardFlowFileRecord.Builder builder = new StandardFlowFileRecord.Builder().fromFlowFile(currRec);
@@ -256,7 +258,7 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
if (claim != null) {
context.getContentRepository().incrementClaimaintCount(claim);
}
- newRecord.setWorking(clone, Collections.<String, String>emptyMap());
+ newRecord.setWorking(clone, Collections.<String, String> emptyMap());
newRecord.setDestination(destination.getFlowFileQueue());
newRecord.setTransferRelationship(record.getTransferRelationship());
@@ -322,9 +324,9 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
final long flowFileLife = System.currentTimeMillis() - flowFile.getEntryDate();
final Connectable connectable = context.getConnectable();
final Object terminator = connectable instanceof ProcessorNode ? ((ProcessorNode) connectable).getProcessor() : connectable;
- LOG.info("{} terminated by {}; life of FlowFile = {} ms", new Object[]{flowFile, terminator, flowFileLife});
+ LOG.info("{} terminated by {}; life of FlowFile = {} ms", new Object[] {flowFile, terminator, flowFileLife});
} else if (record.isWorking() && record.getWorkingClaim() != record.getOriginalClaim()) {
- //records which have been updated - remove original if exists
+ // records which have been updated - remove original if exists
removeContent(record.getOriginalClaim());
}
}
@@ -356,7 +358,7 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
final Map<FlowFileQueue, Collection<FlowFileRecord>> recordMap = new HashMap<>();
for (final StandardRepositoryRecord record : checkpoint.records.values()) {
if (record.isMarkedForAbort() || record.isMarkedForDelete()) {
- continue; //these don't need to be transferred
+ continue; // these don't need to be transferred
}
// record.getCurrent() will return null if this record was created in this session --
// in this case, we just ignore it, and it will be cleaned up by clearing the records map.
@@ -390,7 +392,7 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
if (LOG.isInfoEnabled()) {
final String sessionSummary = summarizeEvents(checkpoint);
if (!sessionSummary.isEmpty()) {
- LOG.info("{} for {}, committed the following events: {}", new Object[]{this, connectableDescription, sessionSummary});
+ LOG.info("{} for {}, committed the following events: {}", new Object[] {this, connectableDescription, sessionSummary});
}
}
@@ -611,9 +613,9 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
boolean creationEventRegistered = false;
if (registeredTypes != null) {
if (registeredTypes.contains(ProvenanceEventType.CREATE)
- || registeredTypes.contains(ProvenanceEventType.FORK)
- || registeredTypes.contains(ProvenanceEventType.JOIN)
- || registeredTypes.contains(ProvenanceEventType.RECEIVE)) {
+ || registeredTypes.contains(ProvenanceEventType.FORK)
+ || registeredTypes.contains(ProvenanceEventType.JOIN)
+ || registeredTypes.contains(ProvenanceEventType.RECEIVE)) {
creationEventRegistered = true;
}
}
@@ -747,7 +749,7 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
}
private StandardProvenanceEventRecord enrich(
- final ProvenanceEventRecord rawEvent, final Map<String, FlowFileRecord> flowFileRecordMap, final Map<FlowFileRecord, StandardRepositoryRecord> records, final boolean updateAttributes) {
+ final ProvenanceEventRecord rawEvent, final Map<String, FlowFileRecord> flowFileRecordMap, final Map<FlowFileRecord, StandardRepositoryRecord> records, final boolean updateAttributes) {
final StandardProvenanceEventRecord.Builder recordBuilder = new StandardProvenanceEventRecord.Builder().fromEvent(rawEvent);
final FlowFileRecord eventFlowFile = flowFileRecordMap.get(rawEvent.getFlowFileUuid());
if (eventFlowFile != null) {
@@ -1039,7 +1041,7 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
final StringBuilder sb = new StringBuilder(512);
if (!LOG.isDebugEnabled() && (largestTransferSetSize > VERBOSE_LOG_THRESHOLD
- || numModified > VERBOSE_LOG_THRESHOLD || numCreated > VERBOSE_LOG_THRESHOLD || numRemoved > VERBOSE_LOG_THRESHOLD)) {
+ || numModified > VERBOSE_LOG_THRESHOLD || numCreated > VERBOSE_LOG_THRESHOLD || numRemoved > VERBOSE_LOG_THRESHOLD)) {
if (numCreated > 0) {
sb.append("created ").append(numCreated).append(" FlowFiles, ");
}
@@ -1097,7 +1099,8 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
private void formatNanos(final long nanos, final StringBuilder sb) {
final long seconds = nanos > 1000000000L ? nanos / 1000000000L : 0L;
- long millis = nanos > 1000000L ? nanos / 1000000L : 0L;;
+ long millis = nanos > 1000000L ? nanos / 1000000L : 0L;
+ ;
final long nanosLeft = nanos % 1000000L;
if (seconds > 0) {
@@ -1272,7 +1275,7 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
int flowFileCount = 0;
long byteCount = 0L;
for (final Connection conn : context.getPollableConnections()) {
- final QueueSize queueSize = conn.getFlowFileQueue().getActiveQueueSize();
+ final QueueSize queueSize = conn.getFlowFileQueue().size();
flowFileCount += queueSize.getObjectCount();
byteCount += queueSize.getByteCount();
}
@@ -1287,8 +1290,8 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
attrs.put(CoreAttributes.UUID.key(), UUID.randomUUID().toString());
final FlowFileRecord fFile = new StandardFlowFileRecord.Builder().id(context.getNextFlowFileSequence())
- .addAttributes(attrs)
- .build();
+ .addAttributes(attrs)
+ .build();
final StandardRepositoryRecord record = new StandardRepositoryRecord(null);
record.setWorking(fFile, attrs);
records.put(fFile, record);
@@ -1324,7 +1327,7 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
context.getContentRepository().incrementClaimaintCount(claim);
}
final StandardRepositoryRecord record = new StandardRepositoryRecord(null);
- record.setWorking(clone, Collections.<String, String>emptyMap());
+ record.setWorking(clone, Collections.<String, String> emptyMap());
records.put(clone, record);
if (offset == 0L && size == example.getSize()) {
@@ -1637,7 +1640,7 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
return;
}
- LOG.info("{} {} FlowFiles have expired and will be removed", new Object[]{this, flowFiles.size()});
+ LOG.info("{} {} FlowFiles have expired and will be removed", new Object[] {this, flowFiles.size()});
final List<RepositoryRecord> expiredRecords = new ArrayList<>(flowFiles.size());
final String processorType;
@@ -1650,7 +1653,7 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
}
final StandardProvenanceReporter expiredReporter = new StandardProvenanceReporter(this, connectable.getIdentifier(),
- processorType, context.getProvenanceRepository(), this);
+ processorType, context.getProvenanceRepository(), this);
final Map<String, FlowFileRecord> recordIdMap = new HashMap<>();
for (final FlowFileRecord flowFile : flowFiles) {
@@ -1664,7 +1667,7 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
final long flowFileLife = System.currentTimeMillis() - flowFile.getEntryDate();
final Object terminator = connectable instanceof ProcessorNode ? ((ProcessorNode) connectable).getProcessor() : connectable;
- LOG.info("{} terminated by {} due to FlowFile expiration; life of FlowFile = {} ms", new Object[]{flowFile, terminator, flowFileLife});
+ LOG.info("{} terminated by {} due to FlowFile expiration; life of FlowFile = {} ms", new Object[] {flowFile, terminator, flowFileLife});
}
try {
@@ -1696,7 +1699,7 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
record.getContentClaimOffset() + claim.getOffset(), record.getSize());
}
- enriched.setAttributes(record.getAttributes(), Collections.<String, String>emptyMap());
+ enriched.setAttributes(record.getAttributes(), Collections.<String, String> emptyMap());
return enriched.build();
}
@@ -1780,9 +1783,9 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
}
try (final InputStream rawIn = getInputStream(source, record.getCurrentClaim(), record.getCurrentClaimOffset());
- final InputStream limitedIn = new LimitedInputStream(rawIn, source.getSize());
- final InputStream disableOnCloseIn = new DisableOnCloseInputStream(limitedIn);
- final ByteCountingInputStream countingStream = new ByteCountingInputStream(disableOnCloseIn, this.bytesRead)) {
+ final InputStream limitedIn = new LimitedInputStream(rawIn, source.getSize());
+ final InputStream disableOnCloseIn = new DisableOnCloseInputStream(limitedIn);
+ final ByteCountingInputStream countingStream = new ByteCountingInputStream(disableOnCloseIn, this.bytesRead)) {
// We want to differentiate between IOExceptions thrown by the repository and IOExceptions thrown from
// Processor code. As a result, as have the FlowFileAccessInputStream that catches IOException from the repository
@@ -1853,7 +1856,7 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
try {
try (final OutputStream rawOut = contentRepo.write(newClaim);
- final OutputStream out = new BufferedOutputStream(rawOut)) {
+ final OutputStream out = new BufferedOutputStream(rawOut)) {
if (header != null && header.length > 0) {
out.write(header);
@@ -2070,10 +2073,10 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
// the original claim if the record is "working" but the content has not been modified
// (e.g., in the case of attributes only were updated)
// In other words:
- // If we modify the attributes of a FlowFile, and then we call record.getWorkingClaim(), this will
- // return the same claim as record.getOriginalClaim(). So we cannot just remove the working claim because
- // that may decrement the original claim (because the 2 claims are the same), and that's NOT what we want to do
- // because we will do that later, in the session.commit() and that would result in removing the original claim twice.
+ // If we modify the attributes of a FlowFile, and then we call record.getWorkingClaim(), this will
+ // return the same claim as record.getOriginalClaim(). So we cannot just remove the working claim because
+ // that may decrement the original claim (because the 2 claims are the same), and that's NOT what we want to do
+ // because we will do that later, in the session.commit() and that would result in removing the original claim twice.
if (contentModified) {
// In this case, it's ok to go ahead and destroy the content because we know that the working claim is going to be
// updated and the given working claim is referenced only by FlowFiles in this session (because it's the Working Claim).
@@ -2196,7 +2199,7 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
@Override
public FlowFile importFrom(final Path source, final boolean keepSourceFile, final FlowFile destination) {
validateRecordState(destination);
- //TODO: find a better solution. With Windows 7 and Java 7 (very early update, at least), Files.isWritable(source.getParent()) returns false, even when it should be true.
+ // TODO: find a better solution. With Windows 7 and Java 7 (very early update, at least), Files.isWritable(source.getParent()) returns false, even when it should be true.
if (!keepSourceFile && !Files.isWritable(source.getParent()) && !source.getParent().toFile().canWrite()) {
// If we do NOT want to keep the file, ensure that we can delete it, or else error.
throw new FlowFileAccessException("Cannot write to path " + source.getParent().toFile().getAbsolutePath() + " so cannot delete file; will not import.");
@@ -2228,9 +2231,9 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
removeTemporaryClaim(record);
final FlowFileRecord newFile = new StandardFlowFileRecord.Builder().fromFlowFile(record.getCurrent())
- .contentClaim(newClaim).contentClaimOffset(claimOffset).size(newSize)
- .addAttribute(CoreAttributes.FILENAME.key(), source.toFile().getName())
- .build();
+ .contentClaim(newClaim).contentClaimOffset(claimOffset).size(newSize)
+ .addAttribute(CoreAttributes.FILENAME.key(), source.toFile().getName())
+ .build();
record.setWorking(newFile, CoreAttributes.FILENAME.key(), source.toFile().getName());
if (!keepSourceFile) {
deleteOnCommit.add(source);
@@ -2370,7 +2373,7 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
*
* @param flowFile the FlowFile to check
* @return <code>true</code> if the FlowFile is known in this session,
- * <code>false</code> otherwise.
+ * <code>false</code> otherwise.
*/
boolean isFlowFileKnown(final FlowFile flowFile) {
return records.containsKey(flowFile);
@@ -2392,8 +2395,8 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
final String key = entry.getKey();
final String value = entry.getValue();
if (CoreAttributes.ALTERNATE_IDENTIFIER.key().equals(key)
- || CoreAttributes.DISCARD_REASON.key().equals(key)
- || CoreAttributes.UUID.key().equals(key)) {
+ || CoreAttributes.DISCARD_REASON.key().equals(key)
+ || CoreAttributes.UUID.key().equals(key)) {
continue;
}
newAttributes.put(key, value);
@@ -2441,10 +2444,10 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
newAttributes.put(CoreAttributes.UUID.key(), UUID.randomUUID().toString());
final FlowFileRecord fFile = new StandardFlowFileRecord.Builder().id(context.getNextFlowFileSequence())
- .addAttributes(newAttributes)
- .lineageIdentifiers(lineageIdentifiers)
- .lineageStartDate(lineageStartDate)
- .build();
+ .addAttributes(newAttributes)
+ .lineageIdentifiers(lineageIdentifiers)
+ .lineageStartDate(lineageStartDate)
+ .build();
final StandardRepositoryRecord record = new StandardRepositoryRecord(null);
record.setWorking(fFile, newAttributes);
@@ -2465,7 +2468,7 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
*/
private static Map<String, String> intersectAttributes(final Collection<FlowFile> flowFileList) {
final Map<String, String> result = new HashMap<>();
- //trivial cases
+ // trivial cases
if (flowFileList == null || flowFileList.isEmpty()) {
return result;
} else if (flowFileList.size() == 1) {
@@ -2478,8 +2481,7 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
*/
final Map<String, String> firstMap = flowFileList.iterator().next().getAttributes();
- outer:
- for (final Map.Entry<String, String> mapEntry : firstMap.entrySet()) {
+ outer: for (final Map.Entry<String, String> mapEntry : firstMap.entrySet()) {
final String key = mapEntry.getKey();
final String value = mapEntry.getValue();
for (final FlowFile flowFile : flowFileList) {
@@ -2539,7 +2541,7 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
private final Set<String> removedFlowFiles = new HashSet<>();
private final Set<String> createdFlowFiles = new HashSet<>();
- private int removedCount = 0; // number of flowfiles removed in this session
+ private int removedCount = 0; // number of flowfiles removed in this session
private long removedBytes = 0L; // size of all flowfiles removed in this session
private long bytesRead = 0L;
private long bytesWritten = 0L;
http://git-wip-us.apache.org/repos/asf/nifi/blob/49a781df/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/util/Connectables.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/util/Connectables.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/util/Connectables.java
index c4d040b..3c4fcdb 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/util/Connectables.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/util/Connectables.java
@@ -26,7 +26,7 @@ public class Connectables {
public static boolean flowFilesQueued(final Connectable connectable) {
for (final Connection conn : connectable.getIncomingConnections()) {
- if (!conn.getFlowFileQueue().isActiveQueueEmpty()) {
+ if (!conn.getFlowFileQueue().isEmpty()) {
return true;
}
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/49a781df/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestFileSystemSwapManager.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestFileSystemSwapManager.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestFileSystemSwapManager.java
index 6eeddc5..f7191c5 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestFileSystemSwapManager.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestFileSystemSwapManager.java
@@ -22,16 +22,26 @@ import java.io.BufferedInputStream;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.nio.file.Files;
+import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
import org.apache.nifi.controller.queue.FlowFileQueue;
import org.apache.nifi.controller.repository.FlowFileRecord;
+import org.apache.nifi.controller.repository.claim.ContentClaim;
import org.apache.nifi.controller.repository.claim.ResourceClaim;
import org.apache.nifi.controller.repository.claim.ResourceClaimManager;
+import org.apache.nifi.flowfile.FlowFile;
import org.junit.Test;
import org.mockito.Mockito;
@@ -47,7 +57,7 @@ public class TestFileSystemSwapManager {
final FlowFileQueue flowFileQueue = Mockito.mock(FlowFileQueue.class);
Mockito.when(flowFileQueue.getIdentifier()).thenReturn("87bb99fe-412c-49f6-a441-d1b0af4e20b4");
- final List<FlowFileRecord> records = FileSystemSwapManager.deserializeFlowFiles(in, flowFileQueue, "/src/test/resources/old-swap-file.swap", new NopResourceClaimManager());
+ final List<FlowFileRecord> records = FileSystemSwapManager.deserializeFlowFiles(in, "/src/test/resources/old-swap-file.swap", flowFileQueue, new NopResourceClaimManager());
assertEquals(10000, records.size());
for (final FlowFileRecord record : records) {
@@ -57,6 +67,53 @@ public class TestFileSystemSwapManager {
}
}
+ @Test
+ public void testRoundTripSerializeDeserialize() throws IOException {
+ final List<FlowFileRecord> toSwap = new ArrayList<>(10000);
+ final Map<String, String> attrs = new HashMap<>();
+ for (int i = 0; i < 10000; i++) {
+ attrs.put("i", String.valueOf(i));
+ final FlowFileRecord ff = new TestFlowFile(attrs, i);
+ toSwap.add(ff);
+ }
+
+ final FlowFileQueue flowFileQueue = Mockito.mock(FlowFileQueue.class);
+ Mockito.when(flowFileQueue.getIdentifier()).thenReturn("87bb99fe-412c-49f6-a441-d1b0af4e20b4");
+
+ final String swapLocation = "target/testRoundTrip.swap";
+ final File swapFile = new File(swapLocation);
+ Files.deleteIfExists(swapFile.toPath());
+
+ try (final FileOutputStream fos = new FileOutputStream(swapFile)) {
+ FileSystemSwapManager.serializeFlowFiles(toSwap, flowFileQueue, swapLocation, fos);
+ }
+
+ final List<FlowFileRecord> swappedIn;
+ try (final FileInputStream fis = new FileInputStream(swapFile);
+ final DataInputStream dis = new DataInputStream(fis)) {
+ swappedIn = FileSystemSwapManager.deserializeFlowFiles(dis, swapLocation, flowFileQueue, Mockito.mock(ResourceClaimManager.class));
+ }
+
+ assertEquals(toSwap.size(), swappedIn.size());
+ for (int i = 0; i < toSwap.size(); i++) {
+ final FlowFileRecord pre = toSwap.get(i);
+ final FlowFileRecord post = swappedIn.get(i);
+
+ assertEquals(pre.getSize(), post.getSize());
+ assertEquals(pre.getAttributes(), post.getAttributes());
+ assertEquals(pre.getSize(), post.getSize());
+ assertEquals(pre.getId(), post.getId());
+ assertEquals(pre.getContentClaim(), post.getContentClaim());
+ assertEquals(pre.getContentClaimOffset(), post.getContentClaimOffset());
+ assertEquals(pre.getEntryDate(), post.getEntryDate());
+ assertEquals(pre.getLastQueueDate(), post.getLastQueueDate());
+ assertEquals(pre.getLineageIdentifiers(), post.getLineageIdentifiers());
+ assertEquals(pre.getLineageStartDate(), post.getLineageStartDate());
+ assertEquals(pre.getPenaltyExpirationMillis(), post.getPenaltyExpirationMillis());
+ }
+ }
+
+
public class NopResourceClaimManager implements ResourceClaimManager {
@Override
@@ -100,4 +157,87 @@ public class TestFileSystemSwapManager {
public void purge() {
}
}
+
+
+ private static class TestFlowFile implements FlowFileRecord {
+ private static final AtomicLong idGenerator = new AtomicLong(0L);
+
+ private final long id = idGenerator.getAndIncrement();
+ private final long entryDate = System.currentTimeMillis();
+ private final long lastQueueDate = System.currentTimeMillis();
+ private final Map<String, String> attributes;
+ private final long size;
+
+
+ public TestFlowFile(final Map<String, String> attributes, final long size) {
+ this.attributes = attributes;
+ this.size = size;
+ }
+
+
+ @Override
+ public long getId() {
+ return id;
+ }
+
+ @Override
+ public long getEntryDate() {
+ return entryDate;
+ }
+
+ @Override
+ public long getLineageStartDate() {
+ return entryDate;
+ }
+
+ @Override
+ public Long getLastQueueDate() {
+ return lastQueueDate;
+ }
+
+ @Override
+ public Set<String> getLineageIdentifiers() {
+ return Collections.emptySet();
+ }
+
+ @Override
+ public boolean isPenalized() {
+ return false;
+ }
+
+ @Override
+ public String getAttribute(String key) {
+ return attributes.get(key);
+ }
+
+ @Override
+ public long getSize() {
+ return size;
+ }
+
+ @Override
+ public Map<String, String> getAttributes() {
+ return Collections.unmodifiableMap(attributes);
+ }
+
+ @Override
+ public int compareTo(final FlowFile o) {
+ return Long.compare(id, o.getId());
+ }
+
+ @Override
+ public long getPenaltyExpirationMillis() {
+ return -1L;
+ }
+
+ @Override
+ public ContentClaim getContentClaim() {
+ return null;
+ }
+
+ @Override
+ public long getContentClaimOffset() {
+ return 0;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/49a781df/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/TestStandardProcessSession.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/TestStandardProcessSession.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/TestStandardProcessSession.java
index 12f8e5e..1783708 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/TestStandardProcessSession.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/TestStandardProcessSession.java
@@ -134,7 +134,7 @@ public class TestStandardProcessSession {
final ProcessScheduler processScheduler = Mockito.mock(ProcessScheduler.class);
final FlowFileSwapManager swapManager = Mockito.mock(FlowFileSwapManager.class);
- flowFileQueue = new StandardFlowFileQueue("1", connection, processScheduler, swapManager, null, 10000);
+ flowFileQueue = new StandardFlowFileQueue("1", connection, flowFileRepo, provenanceRepo, null, processScheduler, swapManager, null, 10000);
when(connection.getFlowFileQueue()).thenReturn(flowFileQueue);
Mockito.doAnswer(new Answer<Object>() {
http://git-wip-us.apache.org/repos/asf/nifi/blob/49a781df/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java
index 8bf5553..0e3bcac 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java
@@ -37,6 +37,11 @@ import java.util.concurrent.TimeUnit;
import javax.ws.rs.WebApplicationException;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.ClassUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.admin.service.UserService;
+import org.apache.nifi.authorization.DownloadAuthorization;
import org.apache.nifi.cluster.protocol.NodeIdentifier;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.connectable.Connectable;
@@ -47,9 +52,10 @@ import org.apache.nifi.controller.ContentAvailability;
import org.apache.nifi.controller.ControllerService;
import org.apache.nifi.controller.Counter;
import org.apache.nifi.controller.FlowController;
-import org.apache.nifi.controller.FlowFileQueue;
import org.apache.nifi.controller.ProcessorNode;
import org.apache.nifi.controller.ScheduledState;
+import org.apache.nifi.controller.queue.FlowFileQueue;
+import org.apache.nifi.controller.queue.QueueSize;
import org.apache.nifi.controller.repository.ContentNotFoundException;
import org.apache.nifi.controller.repository.claim.ContentDirection;
import org.apache.nifi.controller.status.ProcessGroupStatus;
@@ -61,8 +67,8 @@ import org.apache.nifi.groups.ProcessGroupCounts;
import org.apache.nifi.groups.RemoteProcessGroup;
import org.apache.nifi.nar.ExtensionManager;
import org.apache.nifi.nar.NarCloseable;
+import org.apache.nifi.processor.DataUnit;
import org.apache.nifi.processor.Processor;
-import org.apache.nifi.processor.QueueSize;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.provenance.ProvenanceEventRecord;
import org.apache.nifi.provenance.ProvenanceEventRepository;
@@ -75,7 +81,9 @@ import org.apache.nifi.provenance.search.SearchTerm;
import org.apache.nifi.provenance.search.SearchTerms;
import org.apache.nifi.provenance.search.SearchableField;
import org.apache.nifi.remote.RootGroupPort;
+import org.apache.nifi.reporting.BulletinQuery;
import org.apache.nifi.reporting.BulletinRepository;
+import org.apache.nifi.reporting.ComponentType;
import org.apache.nifi.reporting.ReportingTask;
import org.apache.nifi.scheduling.SchedulingStrategy;
import org.apache.nifi.search.SearchContext;
@@ -85,6 +93,7 @@ import org.apache.nifi.services.FlowService;
import org.apache.nifi.user.NiFiUser;
import org.apache.nifi.util.FormatUtils;
import org.apache.nifi.util.NiFiProperties;
+import org.apache.nifi.web.DownloadableContent;
import org.apache.nifi.web.NiFiCoreException;
import org.apache.nifi.web.ResourceNotFoundException;
import org.apache.nifi.web.api.dto.DocumentedTypeDTO;
@@ -104,15 +113,6 @@ import org.apache.nifi.web.api.dto.search.SearchResultsDTO;
import org.apache.nifi.web.api.dto.status.ControllerStatusDTO;
import org.apache.nifi.web.api.dto.status.ProcessGroupStatusDTO;
import org.apache.nifi.web.api.dto.status.StatusHistoryDTO;
-import org.apache.nifi.web.DownloadableContent;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.ClassUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.nifi.admin.service.UserService;
-import org.apache.nifi.authorization.DownloadAuthorization;
-import org.apache.nifi.processor.DataUnit;
-import org.apache.nifi.reporting.BulletinQuery;
-import org.apache.nifi.reporting.ComponentType;
import org.apache.nifi.web.security.user.NiFiUserUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -501,7 +501,7 @@ public class ControllerFacade {
* Site-to-Site communications
*
* @return the socket port that the Cluster Manager is listening on for
- * Site-to-Site communications
+ * Site-to-Site communications
*/
public Integer getClusterManagerRemoteSiteListeningPort() {
return flowController.getClusterManagerRemoteSiteListeningPort();
@@ -512,7 +512,7 @@ public class ControllerFacade {
* Manager are secure
*
* @return whether or not Site-to-Site communications with the Cluster
- * Manager are secure
+ * Manager are secure
*/
public Boolean isClusterManagerRemoteSiteCommsSecure() {
return flowController.isClusterManagerRemoteSiteCommsSecure();
@@ -523,7 +523,7 @@ public class ControllerFacade {
* Site-to-Site communications
*
* @return the socket port that the local instance is listening on for
- * Site-to-Site communications
+ * Site-to-Site communications
*/
public Integer getRemoteSiteListeningPort() {
return flowController.getRemoteSiteListeningPort();
@@ -534,7 +534,7 @@ public class ControllerFacade {
* instance are secure
*
* @return whether or not Site-to-Site communications with the local
- * instance are secure
+ * instance are secure
*/
public Boolean isRemoteSiteCommsSecure() {
return flowController.isRemoteSiteCommsSecure();
[09/30] nifi git commit: NIFI-730: Added additional parameters to
dropFlowFiles
Posted by ma...@apache.org.
NIFI-730: Added additional parameters to dropFlowFiles
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/af78354d
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/af78354d
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/af78354d
Branch: refs/heads/master
Commit: af78354d84dcd72a23f1101567729a04ee008110
Parents: 09d6fe5
Author: Mark Payne <ma...@hotmail.com>
Authored: Tue Oct 13 12:20:18 2015 -0400
Committer: Mark Payne <ma...@hotmail.com>
Committed: Tue Oct 13 12:20:18 2015 -0400
----------------------------------------------------------------------
.../controller/queue/DropFlowFileStatus.java | 6 +++++
.../nifi/controller/queue/FlowFileQueue.java | 5 +++-
.../apache/nifi/controller/queue/QueueSize.java | 14 ++++++++++
.../nifi/controller/DropFlowFileRequest.java | 10 ++++++++
.../nifi/controller/StandardFlowFileQueue.java | 27 ++++++++++----------
5 files changed, 48 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/af78354d/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileStatus.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileStatus.java b/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileStatus.java
index 3c3be9b..7d5b9c2 100644
--- a/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileStatus.java
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileStatus.java
@@ -61,6 +61,12 @@ public interface DropFlowFileStatus {
QueueSize getCurrentSize();
/**
+ * @return a QueueSize representing the number of FlowFiles that have been dropped for this request
+ * and the aggregate size of those FlowFiles
+ */
+ QueueSize getDroppedSize();
+
+ /**
* @return the current state of the operation
*/
DropFlowFileState getState();
http://git-wip-us.apache.org/repos/asf/nifi/blob/af78354d/nifi-api/src/main/java/org/apache/nifi/controller/queue/FlowFileQueue.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/queue/FlowFileQueue.java b/nifi-api/src/main/java/org/apache/nifi/controller/queue/FlowFileQueue.java
index bc2f358..2d67d58 100644
--- a/nifi-api/src/main/java/org/apache/nifi/controller/queue/FlowFileQueue.java
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/queue/FlowFileQueue.java
@@ -181,9 +181,12 @@ public interface FlowFileQueue {
* passed to the {@link #getDropFlowFileStatus(String)} and {@link #cancelDropFlowFileStatus(String)}
* methods in order to obtain the status later or cancel a request
*
+ * @param requestor the entity that is requesting that the FlowFiles be dropped; this will be
+ * included in the Provenance Events that are generated.
+ *
* @return the status of the drop request.
*/
- DropFlowFileStatus dropFlowFiles();
+ DropFlowFileStatus dropFlowFiles(String requestIdentifier, String requestor);
/**
* Returns the current status of a Drop FlowFile Request that was initiated via the
http://git-wip-us.apache.org/repos/asf/nifi/blob/af78354d/nifi-api/src/main/java/org/apache/nifi/controller/queue/QueueSize.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/queue/QueueSize.java b/nifi-api/src/main/java/org/apache/nifi/controller/queue/QueueSize.java
index 528d652..7998d37 100644
--- a/nifi-api/src/main/java/org/apache/nifi/controller/queue/QueueSize.java
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/queue/QueueSize.java
@@ -48,6 +48,20 @@ public class QueueSize {
return totalSizeBytes;
}
+ /**
+ * Returns a new QueueSize that is the sum of this QueueSize and the provided QueueSize
+ *
+ * @param other the other QueueSize to add to this QueueSize
+ * @return a new QueueSize that is the sum of this QueueSize and the provided QueueSize
+ */
+ public QueueSize add(final QueueSize other) {
+ if (other == null) {
+ return new QueueSize(objectCount, totalSizeBytes);
+ }
+
+ return new QueueSize(objectCount + other.getObjectCount(), totalSizeBytes + other.getByteCount());
+ }
+
@Override
public String toString() {
return "QueueSize[FlowFiles=" + objectCount + ", ContentSize=" + NumberFormat.getNumberInstance().format(totalSizeBytes) + " Bytes]";
http://git-wip-us.apache.org/repos/asf/nifi/blob/af78354d/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/DropFlowFileRequest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/DropFlowFileRequest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/DropFlowFileRequest.java
index 609fe75..4104308 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/DropFlowFileRequest.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/DropFlowFileRequest.java
@@ -27,6 +27,7 @@ public class DropFlowFileRequest implements DropFlowFileStatus {
private volatile QueueSize originalSize;
private volatile QueueSize currentSize;
+ private volatile QueueSize droppedSize = new QueueSize(0, 0L);
private volatile long lastUpdated = System.currentTimeMillis();
private volatile Thread executionThread;
@@ -66,6 +67,15 @@ public class DropFlowFileRequest implements DropFlowFileStatus {
}
@Override
+ public QueueSize getDroppedSize() {
+ return droppedSize;
+ }
+
+ void setDroppedSize(final QueueSize droppedSize) {
+ this.droppedSize = droppedSize;
+ }
+
+ @Override
public DropFlowFileState getState() {
return state;
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/af78354d/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
index 073e5fb..b699ceb 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
@@ -29,7 +29,6 @@ import java.util.Map;
import java.util.PriorityQueue;
import java.util.Queue;
import java.util.Set;
-import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
@@ -900,7 +899,7 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
private final ConcurrentMap<String, DropFlowFileRequest> dropRequestMap = new ConcurrentHashMap<>();
@Override
- public DropFlowFileStatus dropFlowFiles() {
+ public DropFlowFileStatus dropFlowFiles(final String requestIdentifier, final String requestor) {
// purge any old requests from the map just to keep it clean. But if there are very requests, which is usually the case, then don't bother
if (dropRequestMap.size() > 10) {
final List<String> toDrop = new ArrayList<>();
@@ -918,10 +917,6 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
}
}
- // TODO: get user name!
- final String userName = null;
-
- final String requestIdentifier = UUID.randomUUID().toString();
final DropFlowFileRequest dropRequest = new DropFlowFileRequest(requestIdentifier);
final Thread t = new Thread(new Runnable() {
@Override
@@ -932,20 +927,23 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
try {
final List<FlowFileRecord> activeQueueRecords = new ArrayList<>(activeQueue);
- drop(activeQueueRecords, userName);
+ QueueSize droppedSize = drop(activeQueueRecords, requestor);
activeQueue.clear();
dropRequest.setCurrentSize(getQueueSize());
+ dropRequest.setDroppedSize(dropRequest.getDroppedSize().add(droppedSize));
- drop(swapQueue, userName);
+ droppedSize = drop(swapQueue, requestor);
swapQueue.clear();
dropRequest.setCurrentSize(getQueueSize());
+ dropRequest.setDroppedSize(dropRequest.getDroppedSize().add(droppedSize));
final Iterator<String> swapLocationItr = swapLocations.iterator();
while (swapLocationItr.hasNext()) {
final String swapLocation = swapLocationItr.next();
final List<FlowFileRecord> swappedIn = swapManager.swapIn(swapLocation, StandardFlowFileQueue.this);
try {
- drop(swappedIn, userName);
+ droppedSize = drop(swappedIn, requestor);
+ dropRequest.setDroppedSize(dropRequest.getDroppedSize().add(droppedSize));
} catch (final Exception e) {
activeQueue.addAll(swappedIn); // ensure that we don't lose the FlowFiles from our queue.
throw e;
@@ -974,15 +972,16 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
return dropRequest;
}
- private void drop(final List<FlowFileRecord> flowFiles, final String user) throws IOException {
+ private QueueSize drop(final List<FlowFileRecord> flowFiles, final String requestor) throws IOException {
// Create a Provenance Event and a FlowFile Repository record for each FlowFile
final List<ProvenanceEventRecord> provenanceEvents = new ArrayList<>(flowFiles.size());
final List<RepositoryRecord> flowFileRepoRecords = new ArrayList<>(flowFiles.size());
for (final FlowFileRecord flowFile : flowFiles) {
- provenanceEvents.add(createDropEvent(flowFile, user));
+ provenanceEvents.add(createDropEvent(flowFile, requestor));
flowFileRepoRecords.add(createDeleteRepositoryRecord(flowFile));
}
+ long dropContentSize = 0L;
for (final FlowFileRecord flowFile : flowFiles) {
final ContentClaim contentClaim = flowFile.getContentClaim();
if (contentClaim == null) {
@@ -995,20 +994,22 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
}
resourceClaimManager.decrementClaimantCount(resourceClaim);
+ dropContentSize += flowFile.getSize();
}
provRepository.registerEvents(provenanceEvents);
flowFileRepository.updateRepository(flowFileRepoRecords);
+ return new QueueSize(flowFiles.size(), dropContentSize);
}
- private ProvenanceEventRecord createDropEvent(final FlowFileRecord flowFile, final String user) {
+ private ProvenanceEventRecord createDropEvent(final FlowFileRecord flowFile, final String requestor) {
final ProvenanceEventBuilder builder = provRepository.eventBuilder();
builder.fromFlowFile(flowFile);
builder.setEventType(ProvenanceEventType.DROP);
builder.setLineageStartDate(flowFile.getLineageStartDate());
builder.setComponentId(getIdentifier());
builder.setComponentType("Connection");
- builder.setDetails("FlowFile manually dropped by user " + user);
+ builder.setDetails("FlowFile manually dropped; request made by " + requestor);
return builder.build();
}
[26/30] nifi git commit: NIFI-730: Do not require a Read Lock in
order to obtain backpressure configuration values for FlowFileQueue's
Posted by ma...@apache.org.
NIFI-730: Do not require a Read Lock in order to obtain backpressure configuration values for FlowFileQueue's
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/edf238e0
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/edf238e0
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/edf238e0
Branch: refs/heads/master
Commit: edf238e004e45566374fcb520555cf4cc30ed8f7
Parents: 570202e
Author: Mark Payne <ma...@hotmail.com>
Authored: Wed Oct 28 10:27:58 2015 -0400
Committer: Mark Payne <ma...@hotmail.com>
Committed: Wed Oct 28 10:27:58 2015 -0400
----------------------------------------------------------------------
.../nifi/controller/StandardFlowFileQueue.java | 27 +++++---------------
1 file changed, 6 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/edf238e0/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
index ddb8225..0f6d258 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
@@ -82,18 +82,12 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
private PriorityQueue<FlowFileRecord> activeQueue = null;
private ArrayList<FlowFileRecord> swapQueue = null;
- // private final AtomicInteger activeQueueSizeRef = new AtomicInteger(0);
- // private long activeQueueContentSize = 0L;
- // private int swappedRecordCount = 0;
- // private long swappedContentSize = 0L;
- // private final AtomicReference<QueueSize> unacknowledgedSizeRef = new AtomicReference<>(new QueueSize(0, 0L));
-
private final AtomicReference<FlowFileQueueSize> size = new AtomicReference<>(new FlowFileQueueSize(0, 0L, 0, 0L, 0, 0L));
- private String maximumQueueDataSize;
- private long maximumQueueByteCount;
private boolean swapMode = false;
- private long maximumQueueObjectCount;
+ private volatile String maximumQueueDataSize;
+ private volatile long maximumQueueByteCount;
+ private volatile long maximumQueueObjectCount;
private final EventReporter eventReporter;
private final AtomicLong flowFileExpirationMillis;
@@ -104,6 +98,7 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
private final int swapThreshold;
private final FlowFileSwapManager swapManager;
private final List<String> swapLocations = new ArrayList<>();
+ @SuppressWarnings("unused")
private final TimedLock readLock;
private final TimedLock writeLock;
private final String identifier;
@@ -178,12 +173,7 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
@Override
public long getBackPressureObjectThreshold() {
- readLock.lock();
- try {
- return maximumQueueObjectCount;
- } finally {
- readLock.unlock("getBackPressureObjectThreshold");
- }
+ return maximumQueueObjectCount;
}
@Override
@@ -200,12 +190,7 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
@Override
public String getBackPressureDataSizeThreshold() {
- readLock.lock();
- try {
- return maximumQueueDataSize;
- } finally {
- readLock.unlock("getBackPressureDataSizeThreshold");
- }
+ return maximumQueueDataSize;
}
@Override
[18/30] nifi git commit: NIFI-730: - Adding progress bar for showing
current status of drop request. - Allowing the user to cancel the current
drop request.
Posted by ma...@apache.org.
NIFI-730:
- Adding progress bar for showing current status of drop request.
- Allowing the user to cancel the current drop request.
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/cad0e7cf
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/cad0e7cf
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/cad0e7cf
Branch: refs/heads/master
Commit: cad0e7cf0f7b5c8f0d4607c55a514e419a1caafc
Parents: a2ae99f
Author: Matt Gilman <ma...@gmail.com>
Authored: Wed Oct 14 12:39:19 2015 -0400
Committer: Matt Gilman <ma...@gmail.com>
Committed: Wed Oct 14 12:39:19 2015 -0400
----------------------------------------------------------------------
.../apache/nifi/web/api/dto/DropRequestDTO.java | 75 ++++--
.../org/apache/nifi/web/NiFiServiceFacade.java | 3 +-
.../nifi/web/StandardNiFiServiceFacade.java | 4 +-
.../apache/nifi/web/api/ConnectionResource.java | 4 +-
.../org/apache/nifi/web/api/dto/DtoFactory.java | 12 +-
.../org/apache/nifi/web/dao/ConnectionDAO.java | 3 +-
.../web/dao/impl/StandardConnectionDAO.java | 4 +-
.../src/main/webapp/WEB-INF/pages/canvas.jsp | 1 +
.../canvas/drop-request-status-dialog.jsp | 29 +++
.../nifi-web-ui/src/main/webapp/css/dialog.css | 25 ++
.../src/main/webapp/images/iconEmptyQueue.png | Bin 0 -> 785 bytes
.../src/main/webapp/js/nf/canvas/nf-actions.js | 212 +++++++++++++----
.../src/main/webapp/js/nf/canvas/nf-canvas.js | 235 ++++++++-----------
.../main/webapp/js/nf/canvas/nf-context-menu.js | 6 +-
14 files changed, 407 insertions(+), 206 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/cad0e7cf/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/DropRequestDTO.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/DropRequestDTO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/DropRequestDTO.java
index f022469..93dc7dd 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/DropRequestDTO.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/DropRequestDTO.java
@@ -38,11 +38,14 @@ public class DropRequestDTO {
private Boolean finished;
private Integer currentCount;
- private String currentSize;
+ private Long currentSize;
+ private String current;
private Integer originalCount;
- private String originalSize;
+ private Long originalSize;
+ private String original;
private Integer droppedCount;
- private String droppedSize;
+ private Long droppedSize;
+ private String dropped;
private String state;
@@ -151,20 +154,34 @@ public class DropRequestDTO {
}
/**
- * @return the siez of the flow files currently queued.
+ * @return the size of the flow files currently queued in bytes.
*/
@ApiModelProperty(
- value = "The size of flow files currently queued."
+ value = "The size of flow files currently queued in bytes."
)
- public String getCurrentSize() {
+ public Long getCurrentSize() {
return currentSize;
}
- public void setCurrentSize(String currentSize) {
+ public void setCurrentSize(Long currentSize) {
this.currentSize = currentSize;
}
/**
+ * @return the count and size of the currently queued flow files.
+ */
+ @ApiModelProperty(
+ value = "The count and size of flow files currently queued."
+ )
+ public String getCurrent() {
+ return current;
+ }
+
+ public void setCurrent(String current) {
+ this.current = current;
+ }
+
+ /**
* @return the number of flow files to be dropped as a result of this request.
*/
@ApiModelProperty(
@@ -179,20 +196,34 @@ public class DropRequestDTO {
}
/**
- * @return the size of the flow files to be dropped as a result of this request.
+ * @return the size of the flow files to be dropped as a result of this request in bytes.
*/
@ApiModelProperty(
- value = "The size of flow files to be dropped as a result of this request."
+ value = "The size of flow files to be dropped as a result of this request in bytes."
)
- public String getOriginalSize() {
+ public Long getOriginalSize() {
return originalSize;
}
- public void setOriginalSize(String originalSize) {
+ public void setOriginalSize(Long originalSize) {
this.originalSize = originalSize;
}
/**
+ * @return the count and size of flow files to be dropped as a result of this request.
+ */
+ @ApiModelProperty(
+ value = "The count and size of flow files to be dropped as a result of this request."
+ )
+ public String getOriginal() {
+ return original;
+ }
+
+ public void setOriginal(String original) {
+ this.original = original;
+ }
+
+ /**
* @return the number of flow files that have been dropped thus far.
*/
@ApiModelProperty(
@@ -207,20 +238,34 @@ public class DropRequestDTO {
}
/**
- * @return the size of the flow files that have been dropped thus far.
+ * @return the size of the flow files that have been dropped thus far in bytes.
*/
@ApiModelProperty(
- value = "The size of flow files that have been dropped thus far."
+ value = "The size of flow files that have been dropped thus far in bytes."
)
- public String getDroppedSize() {
+ public Long getDroppedSize() {
return droppedSize;
}
- public void setDroppedSize(String droppedSize) {
+ public void setDroppedSize(Long droppedSize) {
this.droppedSize = droppedSize;
}
/**
+ * @return the count and size of the flow files that have been dropped thus far.
+ */
+ @ApiModelProperty(
+ value = "The count and size of flow files that have been dropped thus far."
+ )
+ public String getDropped() {
+ return dropped;
+ }
+
+ public void setDropped(String dropped) {
+ this.dropped = dropped;
+ }
+
+ /**
* @return the current state of the drop request.
*/
@ApiModelProperty(
http://git-wip-us.apache.org/repos/asf/nifi/blob/cad0e7cf/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java
index 0703a41..2d3355a 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java
@@ -552,8 +552,9 @@ public interface NiFiServiceFacade {
* @param groupId group
* @param connectionId The ID of the connection
* @param dropRequestId The flow file drop request
+ * @return The DropRequest
*/
- void deleteFlowFileDropRequest(String groupId, String connectionId, String dropRequestId);
+ DropRequestDTO deleteFlowFileDropRequest(String groupId, String connectionId, String dropRequestId);
// ----------------------------------------
// InputPort methods
http://git-wip-us.apache.org/repos/asf/nifi/blob/cad0e7cf/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java
index b2e6000..51f8751 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java
@@ -810,8 +810,8 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade {
}
@Override
- public void deleteFlowFileDropRequest(String groupId, String connectionId, String dropRequestId) {
- connectionDAO.deleteFlowFileDropRequest(groupId, connectionId, dropRequestId);
+ public DropRequestDTO deleteFlowFileDropRequest(String groupId, String connectionId, String dropRequestId) {
+ return dtoFactory.createDropRequestDTO(connectionDAO.deleteFlowFileDropRequest(groupId, connectionId, dropRequestId));
}
@Override
http://git-wip-us.apache.org/repos/asf/nifi/blob/cad0e7cf/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectionResource.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectionResource.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectionResource.java
index 0d00468..b170d39 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectionResource.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectionResource.java
@@ -1086,7 +1086,8 @@ public class ConnectionResource extends ApplicationResource {
}
// delete the drop request
- serviceFacade.deleteFlowFileDropRequest(groupId, connectionId, dropRequestId);
+ final DropRequestDTO dropRequest = serviceFacade.deleteFlowFileDropRequest(groupId, connectionId, dropRequestId);
+ dropRequest.setUri(generateResourceUri("controller", "process-groups", groupId, "connections", connectionId, "contents", "drop-requests", dropRequestId));
// create the revision
final RevisionDTO revision = new RevisionDTO();
@@ -1095,6 +1096,7 @@ public class ConnectionResource extends ApplicationResource {
// create the response entity
final DropRequestEntity entity = new DropRequestEntity();
entity.setRevision(revision);
+ entity.setDropRequest(dropRequest);
return generateOkResponse(entity).build();
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/cad0e7cf/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
index d123704..264268b 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
@@ -312,13 +312,16 @@ public final class DtoFactory {
final QueueSize dropped = dropRequest.getDroppedSize();
dto.setDroppedCount(dropped.getObjectCount());
- dto.setDroppedSize(FormatUtils.formatDataSize(dropped.getByteCount()));
+ dto.setDroppedSize(dropped.getByteCount());
+ dto.setDropped(FormatUtils.formatCount(dropped.getObjectCount()) + " / " + FormatUtils.formatDataSize(dropped.getByteCount()));
if (dropRequest.getOriginalSize() != null) {
final QueueSize original = dropRequest.getOriginalSize();
dto.setOriginalCount(original.getObjectCount());
- dto.setOriginalSize(FormatUtils.formatDataSize(original.getByteCount()));
- dto.setPercentCompleted((dropped.getObjectCount() * 100 ) / original.getObjectCount());
+ dto.setOriginalSize(original.getByteCount());
+ dto.setOriginal(FormatUtils.formatCount(original.getObjectCount()) + " / " + FormatUtils.formatDataSize(original.getByteCount()));
+
+ dto.setPercentCompleted((dropped.getObjectCount() * 100) / original.getObjectCount());
} else {
dto.setPercentCompleted(0);
}
@@ -326,7 +329,8 @@ public final class DtoFactory {
if (dropRequest.getCurrentSize() != null) {
final QueueSize current = dropRequest.getCurrentSize();
dto.setCurrentCount(current.getObjectCount());
- dto.setCurrentSize(FormatUtils.formatDataSize(current.getByteCount()));
+ dto.setCurrentSize(current.getByteCount());
+ dto.setCurrent(FormatUtils.formatCount(current.getObjectCount()) + " / " + FormatUtils.formatDataSize(current.getByteCount()));
}
return dto;
http://git-wip-us.apache.org/repos/asf/nifi/blob/cad0e7cf/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectionDAO.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectionDAO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectionDAO.java
index 3665a4e..2be4403 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectionDAO.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ConnectionDAO.java
@@ -134,6 +134,7 @@ public interface ConnectionDAO {
* @param groupId group id
* @param id The id of the connection
* @param dropRequestId The drop request id
+ * @return The drop request
*/
- void deleteFlowFileDropRequest(String groupId, String id, String dropRequestId);
+ DropFlowFileStatus deleteFlowFileDropRequest(String groupId, String id, String dropRequestId);
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/cad0e7cf/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java
index 399ffa1..66730a3 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java
@@ -496,10 +496,10 @@ public class StandardConnectionDAO extends ComponentDAO implements ConnectionDAO
}
@Override
- public void deleteFlowFileDropRequest(String groupId, String connectionId, String dropRequestId) {
+ public DropFlowFileStatus deleteFlowFileDropRequest(String groupId, String connectionId, String dropRequestId) {
final Connection connection = locateConnection(groupId, connectionId);
final FlowFileQueue queue = connection.getFlowFileQueue();
- queue.cancelDropFlowFileRequest(dropRequestId);
+ return queue.cancelDropFlowFileRequest(dropRequestId);
}
/* setters */
http://git-wip-us.apache.org/repos/asf/nifi/blob/cad0e7cf/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/canvas.jsp
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/canvas.jsp b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/canvas.jsp
index c81bb9d..1bf63a6 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/canvas.jsp
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/canvas.jsp
@@ -119,6 +119,7 @@
<jsp:include page="/WEB-INF/partials/canvas/secure-port-details.jsp"/>
<jsp:include page="/WEB-INF/partials/canvas/label-configuration.jsp"/>
<jsp:include page="/WEB-INF/partials/canvas/connection-configuration.jsp"/>
+ <jsp:include page="/WEB-INF/partials/canvas/drop-request-status-dialog.jsp"/>
<jsp:include page="/WEB-INF/partials/connection-details.jsp"/>
<div id="faded-background"></div>
<div id="glass-pane"></div>
http://git-wip-us.apache.org/repos/asf/nifi/blob/cad0e7cf/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/drop-request-status-dialog.jsp
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/drop-request-status-dialog.jsp b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/drop-request-status-dialog.jsp
new file mode 100644
index 0000000..b001a1a
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/drop-request-status-dialog.jsp
@@ -0,0 +1,29 @@
+<%--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+--%>
+<%@ page contentType="text/html" pageEncoding="UTF-8" session="false" %>
+<div id="drop-request-status-dialog">
+ <div class="dialog-content">
+ <div class="setting">
+ <div class="setting-field">
+ <div id="drop-request-status-message"></div>
+ </div>
+ <div class="setting-field">
+ <div id="drop-request-percent-complete"></div>
+ </div>
+ </div>
+ </div>
+</div>
http://git-wip-us.apache.org/repos/asf/nifi/blob/cad0e7cf/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/dialog.css
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/dialog.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/dialog.css
index 88555e7..6cb6dba 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/dialog.css
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/dialog.css
@@ -269,6 +269,31 @@ div.go-to-link {
height: 195px;
}
+#drop-request-status-dialog {
+ display: none;
+ width: 400px;
+ height: 135px;
+ z-index: 1301;
+}
+
+#drop-request-percent-complete {
+ width: 378px;
+ border-radius: 0;
+}
+
+#drop-request-percent-complete .ui-progressbar-value {
+ border-radius: 0;
+}
+
+div.progress-label {
+ color: #000000;
+ display: block;
+ font-weight: bold;
+ text-align: center;
+ width: 378px;
+ margin-top: 4px;
+}
+
#nf-ok-dialog {
z-index: 1302;
display: none;
http://git-wip-us.apache.org/repos/asf/nifi/blob/cad0e7cf/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/images/iconEmptyQueue.png
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/images/iconEmptyQueue.png b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/images/iconEmptyQueue.png
new file mode 100644
index 0000000..42e5a15
Binary files /dev/null and b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/images/iconEmptyQueue.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/cad0e7cf/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js
index 2a94ec5..ba144ef 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js
@@ -24,6 +24,31 @@ nf.Actions = (function () {
controller: '../nifi-api/controller'
}
};
+
+ /**
+ * Initializes the drop request status dialog.
+ */
+ var initializeDropRequestStatusDialog = function () {
+ // initialize the drop requst progress bar
+ var dropRequestProgressBar = $('#drop-request-percent-complete').progressbar();
+
+ // configure the drop request status dialog
+ $('#drop-request-status-dialog').modal({
+ headerText: 'Emptying queue',
+ overlayBackground: false,
+ handler: {
+ close: function () {
+ // reset the progress bar
+ dropRequestProgressBar.find('div.progress-label').remove();
+
+ // update the progress bar
+ var label = $('<div class="progress-label"></div>').text('0%');
+ dropRequestProgressBar.progressbar('value', 0).append(label);
+ }
+ }
+ });
+ };
+
/**
* Updates the resource with the specified data.
@@ -79,6 +104,13 @@ nf.Actions = (function () {
return {
/**
+ * Initializes the actions.
+ */
+ init: function () {
+ initializeDropRequestStatusDialog();
+ },
+
+ /**
* Enters the specified process group.
*
* @param {selection} selection The the currently selected component
@@ -855,56 +887,144 @@ nf.Actions = (function () {
return;
}
- var MAX_DELAY = 4;
-
- // process the drop request
- var processDropRequest = function (dropRequest, nextDelay) {
- // see if the drop request has completed
- if (dropRequest.finished === true) {
- // request is finished so it can be removed
- deleteDropRequest(dropRequest);
- } else {
- // update the UI with the current status of the drop request
- schedule(dropRequest, nextDelay);
- }
- };
+ // prompt the user before emptying the queue
+ nf.Dialog.showYesNoDialog({
+ dialogContent: 'Are you sure you want to empty the queue? All data enqueued at the time of the request will be removed from the dataflow.',
+ overlayBackground: false,
+ yesHandler: function () {
+ // get the connection data
+ var connection = selection.datum();
+
+ var MAX_DELAY = 4;
+ var cancelled = false;
+ var dropRequest = null;
+ var dropRequestTimer = null;
+
+ // updates the progress bar
+ var updateProgress = function (percentComplete) {
+ // remove existing labels
+ var progressBar = $('#drop-request-percent-complete');
+ progressBar.find('div.progress-label').remove();
+
+ // update the progress bar
+ var label = $('<div class="progress-label"></div>').text(percentComplete + '%');
+ if (percentComplete > 0) {
+ label.css('margin-top', '-19px');
+ }
+ progressBar.progressbar('value', percentComplete).append(label);
+ };
+
+ // update the button model of the drop request status dialog
+ $('#drop-request-status-dialog').modal('setButtonModel', [{
+ buttonText: 'Cancel',
+ handler: {
+ click: function () {
+ cancelled = true;
+
+ // we are waiting for the next poll attempt
+ if (dropRequestTimer !== null) {
+ // cancel it
+ clearTimeout(dropRequestTimer);
+
+ // cancel the provenance
+ completeDropRequest();
+ }
+ }
+ }
+ }]);
+
+ // completes the drop request by removing it and showing how many flowfiles were deleted
+ var completeDropRequest = function () {
+ if (nf.Common.isDefinedAndNotNull(dropRequest)) {
+ $.ajax({
+ type: 'DELETE',
+ url: dropRequest.uri,
+ dataType: 'json'
+ }).done(function(response) {
+ // report the results of this drop request
+ dropRequest = response.dropRequest;
+
+ // parse the dropped stats to render results
+ var tokens = dropRequest.dropped.split(/ \/ /);
+ var results = $('<div></div>').text('Successfully removed ' + tokens[0] + ' (' + tokens[1] + ') FlowFiles');
+ nf.Dialog.showOkDialog({
+ dialogContent: results,
+ overlayBackground: false
+ });
+ }).always(function() {
+ $('#drop-request-status-dialog').modal('hide');
+ });
+ } else {
+ // nothing was removed
+ nf.Dialog.showYesNoDialog({
+ dialogContent: 'No FlowFiles were removed.',
+ overlayBackground: false
+ });
+
+ // close the dialog
+ $('#drop-request-status-dialog').modal('hide');
+ }
+ };
+
+ // process the drop request
+ var processDropRequest = function (delay) {
+ // update the percent complete
+ updateProgress(dropRequest.percentCompleted);
+
+ // update the status of the drop request
+ $('#drop-request-status-message').text(dropRequest.state);
+
+ // update the current number of enqueued flowfiles
+ if (nf.Common.isDefinedAndNotNull(dropRequest.currentCount)) {
+ connection.status.queued = dropRequest.current;
+ nf.Connection.refresh(connection.id);
+ }
+
+ // close the dialog if the
+ if (dropRequest.finished === true || cancelled === true) {
+ completeDropRequest();
+ } else {
+ // wait delay to poll again
+ dropRequestTimer = setTimeout(function () {
+ // clear the drop request timer
+ dropRequestTimer = null;
+
+ // schedule to poll the status again in nextDelay
+ pollDropRequest(Math.min(MAX_DELAY, delay * 2));
+ }, delay * 1000);
+ }
+ };
- // schedule for the next poll iteration
- var schedule = function (dropRequest, delay) {
- setTimeout(function () {
+ // schedule for the next poll iteration
+ var pollDropRequest = function (nextDelay) {
+ $.ajax({
+ type: 'GET',
+ url: dropRequest.uri,
+ dataType: 'json'
+ }).done(function(response) {
+ dropRequest = response.dropRequest;
+ processDropRequest(nextDelay);
+ }).fail(completeDropRequest);
+ };
+
+ // issue the request to delete the flow files
$.ajax({
- type: 'GET',
- url: dropRequest.uri,
+ type: 'DELETE',
+ url: connection.component.uri + '/contents',
dataType: 'json'
}).done(function(response) {
- var dropRequest = response.dropRequest;
- processDropRequest(dropRequest, Math.min(MAX_DELAY, delay * 2));
- }).fail(nf.Common.handleAjaxError);
- }, delay * 1000);
- };
-
- // delete the drop request
- var deleteDropRequest = function (dropRequest) {
- $.ajax({
- type: 'DELETE',
- url: dropRequest.uri,
- dataType: 'json'
- }).done(function() {
- // drop request has been deleted
- }).fail(nf.Common.handleAjaxError);
- };
-
- // get the connection data
- var connection = selection.datum();
-
- // issue the request to delete the flow files
- $.ajax({
- type: 'DELETE',
- url: connection.component.uri + '/contents',
- dataType: 'json'
- }).done(function(response) {
- processDropRequest(response.dropRequest, 1);
- }).fail(nf.Common.handleAjaxError);
+ // initialize the progress bar value
+ updateProgress(0);
+
+ // show the progress dialog
+ $('#drop-request-status-dialog').modal('show');
+
+ // process the drop request
+ dropRequest = response.dropRequest;
+ processDropRequest(1);
+ }).fail(completeDropRequest);
+ }
+ });
},
/**
http://git-wip-us.apache.org/repos/asf/nifi/blob/cad0e7cf/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-canvas.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-canvas.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-canvas.js
index 33f116e..7137fe4 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-canvas.js
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-canvas.js
@@ -188,7 +188,7 @@ nf.Canvas = (function () {
if (!refreshContainer.is(':visible')) {
$('#stats-last-refreshed').addClass('alert');
var refreshMessage = "This flow has been modified by '" + revision.lastModifier + "'. Please refresh.";
-
+
// update the tooltip
var refreshRequiredIcon = $('#refresh-required-icon');
if (refreshRequiredIcon.data('qtip')) {
@@ -198,10 +198,10 @@ nf.Canvas = (function () {
content: refreshMessage
}, nf.CanvasUtils.config.systemTooltipConfig));
}
-
+
refreshContainer.show();
}
-
+
// insert the refresh needed text in the settings - if necessary
if (!settingsRefreshIcon.is(':visible')) {
$('#settings-last-refreshed').addClass('alert');
@@ -333,7 +333,7 @@ nf.Canvas = (function () {
'offset': '100%',
'stop-color': '#ffffff'
});
-
+
// define the gradient for the expiration icon
var expirationBackground = defs.append('linearGradient')
.attr({
@@ -343,7 +343,7 @@ nf.Canvas = (function () {
'x2': '0%',
'y2': '100%'
});
-
+
expirationBackground.append('stop')
.attr({
'offset': '0%',
@@ -397,105 +397,105 @@ nf.Canvas = (function () {
// prevent further propagation (to parents and others handlers
// on the same element to prevent zoom behavior)
d3.event.stopImmediatePropagation();
-
+
// prevents the browser from changing to a text selection cursor
d3.event.preventDefault();
}
})
- .on('mousemove.selection', function () {
- // update selection box if shift is held down
- if (d3.event.shiftKey) {
- // get the selection box
- var selectionBox = d3.select('rect.selection');
- if (!selectionBox.empty()) {
- // get the original position
- var originalPosition = selectionBox.datum();
- var position = d3.mouse(canvas.node());
-
- var d = {};
- if (originalPosition[0] < position[0]) {
- d.x = originalPosition[0];
- d.width = position[0] - originalPosition[0];
- } else {
- d.x = position[0];
- d.width = originalPosition[0] - position[0];
- }
-
- if (originalPosition[1] < position[1]) {
- d.y = originalPosition[1];
- d.height = position[1] - originalPosition[1];
- } else {
- d.y = position[1];
- d.height = originalPosition[1] - position[1];
- }
+ .on('mousemove.selection', function () {
+ // update selection box if shift is held down
+ if (d3.event.shiftKey) {
+ // get the selection box
+ var selectionBox = d3.select('rect.selection');
+ if (!selectionBox.empty()) {
+ // get the original position
+ var originalPosition = selectionBox.datum();
+ var position = d3.mouse(canvas.node());
+
+ var d = {};
+ if (originalPosition[0] < position[0]) {
+ d.x = originalPosition[0];
+ d.width = position[0] - originalPosition[0];
+ } else {
+ d.x = position[0];
+ d.width = originalPosition[0] - position[0];
+ }
- // update the selection box
- selectionBox.attr(d);
-
- // prevent further propagation (to parents)
- d3.event.stopPropagation();
- }
- }
- })
- .on('mouseup.selection', function () {
- // ensure this originated from clicking the canvas, not a component.
- // when clicking on a component, the event propagation is stopped so
- // it never reaches the canvas. we cannot do this however on up events
- // since the drag events break down
- if (canvasClicked === false) {
- return;
- }
+ if (originalPosition[1] < position[1]) {
+ d.y = originalPosition[1];
+ d.height = position[1] - originalPosition[1];
+ } else {
+ d.y = position[1];
+ d.height = originalPosition[1] - position[1];
+ }
- // reset the canvas click flag
- canvasClicked = false;
-
- // get the selection box
- var selectionBox = d3.select('rect.selection');
- if (!selectionBox.empty()) {
- var selectionBoundingBox = {
- x: parseInt(selectionBox.attr('x'), 10),
- y: parseInt(selectionBox.attr('y'), 10),
- width: parseInt(selectionBox.attr('width'), 10),
- height: parseInt(selectionBox.attr('height'), 10)
- };
+ // update the selection box
+ selectionBox.attr(d);
- // see if a component should be selected or not
- d3.selectAll('g.component').classed('selected', function (d) {
- // consider it selected if its already selected or enclosed in the bounding box
- return d3.select(this).classed('selected') ||
- d.component.position.x >= selectionBoundingBox.x && (d.component.position.x + d.dimensions.width) <= (selectionBoundingBox.x + selectionBoundingBox.width) &&
- d.component.position.y >= selectionBoundingBox.y && (d.component.position.y + d.dimensions.height) <= (selectionBoundingBox.y + selectionBoundingBox.height);
- });
+ // prevent further propagation (to parents)
+ d3.event.stopPropagation();
+ }
+ }
+ })
+ .on('mouseup.selection', function () {
+ // ensure this originated from clicking the canvas, not a component.
+ // when clicking on a component, the event propagation is stopped so
+ // it never reaches the canvas. we cannot do this however on up events
+ // since the drag events break down
+ if (canvasClicked === false) {
+ return;
+ }
- // see if a connection should be selected or not
- d3.selectAll('g.connection').classed('selected', function (d) {
- // consider all points
- var points = [d.start].concat(d.bends, [d.end]);
+ // reset the canvas click flag
+ canvasClicked = false;
- // determine the bounding box
- var x = d3.extent(points, function (pt) {
- return pt.x;
- });
- var y = d3.extent(points, function (pt) {
- return pt.y;
- });
+ // get the selection box
+ var selectionBox = d3.select('rect.selection');
+ if (!selectionBox.empty()) {
+ var selectionBoundingBox = {
+ x: parseInt(selectionBox.attr('x'), 10),
+ y: parseInt(selectionBox.attr('y'), 10),
+ width: parseInt(selectionBox.attr('width'), 10),
+ height: parseInt(selectionBox.attr('height'), 10)
+ };
- // consider it selected if its already selected or enclosed in the bounding box
- return d3.select(this).classed('selected') ||
- x[0] >= selectionBoundingBox.x && x[1] <= (selectionBoundingBox.x + selectionBoundingBox.width) &&
- y[0] >= selectionBoundingBox.y && y[1] <= (selectionBoundingBox.y + selectionBoundingBox.height);
- });
+ // see if a component should be selected or not
+ d3.selectAll('g.component').classed('selected', function (d) {
+ // consider it selected if its already selected or enclosed in the bounding box
+ return d3.select(this).classed('selected') ||
+ d.component.position.x >= selectionBoundingBox.x && (d.component.position.x + d.dimensions.width) <= (selectionBoundingBox.x + selectionBoundingBox.width) &&
+ d.component.position.y >= selectionBoundingBox.y && (d.component.position.y + d.dimensions.height) <= (selectionBoundingBox.y + selectionBoundingBox.height);
+ });
+
+ // see if a connection should be selected or not
+ d3.selectAll('g.connection').classed('selected', function (d) {
+ // consider all points
+ var points = [d.start].concat(d.bends, [d.end]);
+
+ // determine the bounding box
+ var x = d3.extent(points, function (pt) {
+ return pt.x;
+ });
+ var y = d3.extent(points, function (pt) {
+ return pt.y;
+ });
- // remove the selection box
- selectionBox.remove();
- } else if (panning === false) {
- // deselect as necessary if we are not panning
- nf.CanvasUtils.getSelection().classed('selected', false);
- }
+ // consider it selected if its already selected or enclosed in the bounding box
+ return d3.select(this).classed('selected') ||
+ x[0] >= selectionBoundingBox.x && x[1] <= (selectionBoundingBox.x + selectionBoundingBox.width) &&
+ y[0] >= selectionBoundingBox.y && y[1] <= (selectionBoundingBox.y + selectionBoundingBox.height);
+ });
+
+ // remove the selection box
+ selectionBox.remove();
+ } else if (panning === false) {
+ // deselect as necessary if we are not panning
+ nf.CanvasUtils.getSelection().classed('selected', false);
+ }
- // update the toolbar
- nf.CanvasToolbar.refresh();
- });
+ // update the toolbar
+ nf.CanvasToolbar.refresh();
+ });
// define a function for update the graph dimensions
var updateGraphSize = function () {
@@ -510,7 +510,7 @@ nf.Canvas = (function () {
var top = parseInt(canvasContainer.css('top'), 10);
var windowHeight = $(window).height();
var canvasHeight = (windowHeight - (bottom + top));
-
+
// canvas/svg
canvasContainer.css({
'height': canvasHeight + 'px',
@@ -536,7 +536,7 @@ nf.Canvas = (function () {
}
}).on('keydown', function (evt) {
var isCtrl = evt.ctrlKey || evt.metaKey;
-
+
// consider escape, before checking dialogs
if (!isCtrl && evt.keyCode === 27) {
// esc
@@ -552,7 +552,7 @@ nf.Canvas = (function () {
// first consider read only property detail dialog
if ($('div.property-detail').is(':visible')) {
nf.Common.removeAllPropertyDetailDialogs();
-
+
// prevent further bubbling as we're already handled it
evt.stopPropagation();
evt.preventDefault();
@@ -570,7 +570,7 @@ nf.Canvas = (function () {
var dialogMax = null;
// identify the top most cancellable
- $.each(cancellables, function(_, cancellable) {
+ $.each(cancellables, function (_, cancellable) {
var dialog = $(cancellable);
var zIndex = dialog.css('zIndex');
@@ -615,10 +615,10 @@ nf.Canvas = (function () {
}
}
}
-
+
return;
}
-
+
// if a dialog is open, disable canvas shortcuts
if ($('.dialog').is(':visible')) {
return;
@@ -833,7 +833,7 @@ nf.Canvas = (function () {
bulletinIcon.show();
}
}
-
+
// update controller service and reporting task bulletins
nf.Settings.setBulletins(controllerStatus.controllerServiceBulletins, controllerStatus.reportingTaskBulletins);
@@ -934,21 +934,17 @@ nf.Canvas = (function () {
};
return {
-
CANVAS_OFFSET: 0,
-
/**
* Determines if the current broswer supports SVG.
*/
SUPPORTS_SVG: !!document.createElementNS && !!document.createElementNS('http://www.w3.org/2000/svg', 'svg').createSVGRect,
-
/**
* Hides the splash that is displayed while the application is loading.
*/
hideSplash: function () {
$('#splash').fadeOut();
},
-
/**
* Stop polling for revision.
*/
@@ -956,7 +952,6 @@ nf.Canvas = (function () {
// set polling flag
revisionPolling = false;
},
-
/**
* Remove the status poller.
*/
@@ -964,7 +959,6 @@ nf.Canvas = (function () {
// set polling flag
statusPolling = false;
},
-
/**
* Reloads the flow from the server based on the currently specified group id.
* To load another group, update nf.Canvas.setGroupId and call nf.Canvas.reload.
@@ -973,7 +967,7 @@ nf.Canvas = (function () {
return $.Deferred(function (deferred) {
// hide the context menu
nf.ContextMenu.hide();
-
+
// get the process group to refresh everything
var processGroupXhr = reloadProcessGroup(nf.Canvas.getGroupId());
var statusXhr = reloadFlowStatus();
@@ -1007,7 +1001,6 @@ nf.Canvas = (function () {
});
}).promise();
},
-
/**
* Reloads the status.
*/
@@ -1021,7 +1014,6 @@ nf.Canvas = (function () {
});
}).promise();
},
-
/**
* Initialize NiFi.
*/
@@ -1099,6 +1091,7 @@ nf.Canvas = (function () {
nf.GraphControl.init();
nf.Search.init();
nf.Settings.init();
+ nf.Actions.init();
// initialize the component behaviors
nf.Draggable.init();
@@ -1146,7 +1139,6 @@ nf.Canvas = (function () {
}).fail(nf.Common.handleAjaxError);
}).fail(nf.Common.handleAjaxError);
},
-
/**
* Defines the gradient colors used to render processors.
*
@@ -1155,7 +1147,6 @@ nf.Canvas = (function () {
defineProcessorColors: function (colors) {
setColors(colors, 'processor');
},
-
/**
* Defines the gradient colors used to render label.
*
@@ -1164,7 +1155,6 @@ nf.Canvas = (function () {
defineLabelColors: function (colors) {
setColors(colors, 'label');
},
-
/**
* Return whether this instance of NiFi is clustered.
*
@@ -1173,14 +1163,12 @@ nf.Canvas = (function () {
isClustered: function () {
return clustered === true;
},
-
/**
* Returns whether site to site communications is secure.
*/
isSecureSiteToSite: function () {
return secureSiteToSite;
},
-
/**
* Set the group id.
*
@@ -1189,14 +1177,12 @@ nf.Canvas = (function () {
setGroupId: function (gi) {
groupId = gi;
},
-
/**
* Get the group id.
*/
getGroupId: function () {
return groupId;
},
-
/**
* Set the group name.
*
@@ -1205,14 +1191,12 @@ nf.Canvas = (function () {
setGroupName: function (gn) {
groupName = gn;
},
-
/**
* Get the group name.
*/
getGroupName: function () {
return groupName;
},
-
/**
* Set the parent group id.
*
@@ -1221,16 +1205,14 @@ nf.Canvas = (function () {
setParentGroupId: function (pgi) {
parentGroupId = pgi;
},
-
/**
* Get the parent group id.
*/
getParentGroupId: function () {
return parentGroupId;
},
-
View: (function () {
-
+
/**
* Updates component visibility based on their proximity to the screen's viewport.
*/
@@ -1297,8 +1279,8 @@ nf.Canvas = (function () {
.classed('entering', function () {
return visible && !wasVisible;
}).classed('leaving', function () {
- return !visible && wasVisible;
- });
+ return !visible && wasVisible;
+ });
};
// get the all components
@@ -1385,7 +1367,6 @@ nf.Canvas = (function () {
// add the behavior to the canvas and disable dbl click zoom
svg.call(behavior).on('dblclick.zoom', null);
},
-
/**
* Whether or not a component should be rendered based solely on the current scale.
*
@@ -1394,7 +1375,6 @@ nf.Canvas = (function () {
shouldRenderPerScale: function () {
return nf.Canvas.View.scale() >= MIN_SCALE_TO_RENDER;
},
-
/**
* Updates component visibility based on the current translation/scale.
*/
@@ -1402,7 +1382,6 @@ nf.Canvas = (function () {
updateComponentVisibility();
nf.Graph.pan();
},
-
/**
* Sets/gets the current translation.
*
@@ -1415,7 +1394,6 @@ nf.Canvas = (function () {
behavior.translate(translate);
}
},
-
/**
* Sets/gets the current scale.
*
@@ -1428,7 +1406,6 @@ nf.Canvas = (function () {
behavior.scale(scale);
}
},
-
/**
* Zooms in a single zoom increment.
*/
@@ -1453,7 +1430,6 @@ nf.Canvas = (function () {
height: 1
});
},
-
/**
* Zooms out a single zoom increment.
*/
@@ -1478,7 +1454,6 @@ nf.Canvas = (function () {
height: 1
});
},
-
/**
* Zooms to fit the entire graph on the canvas.
*/
@@ -1525,7 +1500,6 @@ nf.Canvas = (function () {
height: canvasHeight / newScale
});
},
-
/**
* Zooms to the actual size (1 to 1).
*/
@@ -1574,7 +1548,6 @@ nf.Canvas = (function () {
// center as appropriate
nf.CanvasUtils.centerBoundingBox(box);
},
-
/**
* Refreshes the view based on the configured translation and scale.
*
http://git-wip-us.apache.org/repos/asf/nifi/blob/cad0e7cf/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-context-menu.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-context-menu.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-context-menu.js
index f2b358d..2ce3f7d 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-context-menu.js
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-context-menu.js
@@ -278,11 +278,11 @@ nf.ContextMenu = (function () {
};
/**
- * Only DFMs can delete flow files from a connection.
+ * Only DFMs can empty a queue.
*
* @param {selection} selection
*/
- var canDeleteFlowFiles = function (selection) {
+ var canEmptyQueue = function (selection) {
return nf.Common.isDFM() && isConnection(selection);
};
@@ -382,7 +382,7 @@ nf.ContextMenu = (function () {
{condition: isCopyable, menuItem: {img: 'images/iconCopy.png', text: 'Copy', action: 'copy'}},
{condition: isPastable, menuItem: {img: 'images/iconPaste.png', text: 'Paste', action: 'paste'}},
{condition: canMoveToParent, menuItem: {img: 'images/iconMoveToParent.png', text: 'Move to parent group', action: 'moveIntoParent'}},
- {condition: canDeleteFlowFiles, menuItem: {img: 'images/iconDelete.png', text: 'Empty queue', action: 'emptyQueue'}},
+ {condition: canEmptyQueue, menuItem: {img: 'images/iconEmptyQueue.png', text: 'Empty queue', action: 'emptyQueue'}},
{condition: isDeletable, menuItem: {img: 'images/iconDelete.png', text: 'Delete', action: 'delete'}}
];
[17/30] nifi git commit: NIFI-730: Make cancel request actually cancel
Posted by ma...@apache.org.
NIFI-730: Make cancel request actually cancel
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/a2ae99f8
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/a2ae99f8
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/a2ae99f8
Branch: refs/heads/master
Commit: a2ae99f89965a3fe1bd6591204bdb187a377ae1c
Parents: 5867193
Author: Mark Payne <ma...@hotmail.com>
Authored: Wed Oct 14 11:11:48 2015 -0400
Committer: Mark Payne <ma...@hotmail.com>
Committed: Wed Oct 14 11:11:48 2015 -0400
----------------------------------------------------------------------
.../apache/nifi/controller/DropFlowFileRequest.java | 2 +-
.../nifi/controller/StandardFlowFileQueue.java | 15 +++++++++++++++
2 files changed, 16 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/a2ae99f8/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/DropFlowFileRequest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/DropFlowFileRequest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/DropFlowFileRequest.java
index 58695c2..7eea86a 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/DropFlowFileRequest.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/DropFlowFileRequest.java
@@ -77,7 +77,7 @@ public class DropFlowFileRequest implements DropFlowFileStatus {
}
@Override
- public DropFlowFileState getState() {
+ public synchronized DropFlowFileState getState() {
return state;
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/a2ae99f8/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
index 8085760..062c424 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
@@ -952,6 +952,11 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
QueueSize droppedSize;
try {
+ if (dropRequest.getState() == DropFlowFileState.CANCELED) {
+ logger.info("Cancel requested for DropFlowFileRequest {}", requestIdentifier);
+ return;
+ }
+
droppedSize = drop(activeQueueRecords, requestor);
logger.debug("For DropFlowFileRequest {}, Dropped {} from active queue", requestIdentifier, droppedSize);
} catch (final IOException ioe) {
@@ -972,6 +977,11 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
logger.debug("For DropFlowFileRequest {}, Swap Queue has {} elements, Swapped Record Count = {}, Swapped Content Size = {}",
requestIdentifier, swapQueue.size(), swapSize.getObjectCount(), swapSize.getByteCount());
+ if (dropRequest.getState() == DropFlowFileState.CANCELED) {
+ logger.info("Cancel requested for DropFlowFileRequest {}", requestIdentifier);
+ return;
+ }
+
droppedSize = drop(swapQueue, requestor);
} catch (final IOException ioe) {
logger.error("Failed to drop the FlowFiles from queue {} due to {}", StandardFlowFileQueue.this.getIdentifier(), ioe.toString());
@@ -995,6 +1005,11 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
List<FlowFileRecord> swappedIn = null;
try {
+ if (dropRequest.getState() == DropFlowFileState.CANCELED) {
+ logger.info("Cancel requested for DropFlowFileRequest {}", requestIdentifier);
+ return;
+ }
+
swappedIn = swapManager.swapIn(swapLocation, StandardFlowFileQueue.this);
droppedSize = drop(swappedIn, requestor);
} catch (final IOException ioe) {
[11/30] nifi git commit: NIFI-730: Updated queue sizes appropriately
during dropping of flowfiles
Posted by ma...@apache.org.
NIFI-730: Updated queue sizes appropriately during dropping of flowfiles
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/72ff2a25
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/72ff2a25
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/72ff2a25
Branch: refs/heads/master
Commit: 72ff2a25d5455442d5aec27835f953e6ab36eca3
Parents: 4b41aaa
Author: Mark Payne <ma...@hotmail.com>
Authored: Tue Oct 13 15:46:56 2015 -0400
Committer: Mark Payne <ma...@hotmail.com>
Committed: Tue Oct 13 15:46:56 2015 -0400
----------------------------------------------------------------------
.../org/apache/nifi/controller/StandardFlowFileQueue.java | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/72ff2a25/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
index b699ceb..82c1c7e 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
@@ -924,11 +924,14 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
writeLock.lock();
try {
dropRequest.setState(DropFlowFileState.DROPPING_ACTIVE_FLOWFILES);
+ dropRequest.setOriginalSize(getQueueSize());
try {
final List<FlowFileRecord> activeQueueRecords = new ArrayList<>(activeQueue);
QueueSize droppedSize = drop(activeQueueRecords, requestor);
activeQueue.clear();
+ activeQueueContentSize = 0;
+ activeQueueSizeRef.set(0);
dropRequest.setCurrentSize(getQueueSize());
dropRequest.setDroppedSize(dropRequest.getDroppedSize().add(droppedSize));
@@ -936,6 +939,9 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
swapQueue.clear();
dropRequest.setCurrentSize(getQueueSize());
dropRequest.setDroppedSize(dropRequest.getDroppedSize().add(droppedSize));
+ swapMode = false;
+ swappedContentSize -= droppedSize.getByteCount();
+ swappedRecordCount -= droppedSize.getObjectCount();
final Iterator<String> swapLocationItr = swapLocations.iterator();
while (swapLocationItr.hasNext()) {
@@ -943,12 +949,14 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
final List<FlowFileRecord> swappedIn = swapManager.swapIn(swapLocation, StandardFlowFileQueue.this);
try {
droppedSize = drop(swappedIn, requestor);
- dropRequest.setDroppedSize(dropRequest.getDroppedSize().add(droppedSize));
} catch (final Exception e) {
activeQueue.addAll(swappedIn); // ensure that we don't lose the FlowFiles from our queue.
throw e;
}
+ dropRequest.setDroppedSize(dropRequest.getDroppedSize().add(droppedSize));
+ swappedContentSize -= droppedSize.getByteCount();
+ swappedRecordCount -= droppedSize.getObjectCount();
dropRequest.setCurrentSize(getQueueSize());
swapLocationItr.remove();
}
[24/30] nifi git commit: NIFI-730: - Updating the styling of the drop
request status dialog. - Rendering any errors that may have occurred.
Posted by ma...@apache.org.
NIFI-730:
- Updating the styling of the drop request status dialog.
- Rendering any errors that may have occurred.
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/a8724038
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/a8724038
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/a8724038
Branch: refs/heads/master
Commit: a872403831a2e816ae41c5ac11d43f7067ce32cc
Parents: 2b4999c
Author: Matt Gilman <ma...@gmail.com>
Authored: Fri Oct 16 09:37:57 2015 -0400
Committer: Matt Gilman <ma...@gmail.com>
Committed: Fri Oct 16 09:37:57 2015 -0400
----------------------------------------------------------------------
.../apache/nifi/web/api/dto/DropRequestDTO.java | 12 +++++++
.../org/apache/nifi/web/api/dto/DtoFactory.java | 5 ++-
.../nifi-web-ui/src/main/webapp/css/dialog.css | 7 +++-
.../src/main/webapp/js/nf/canvas/nf-actions.js | 37 ++++++++++++++++----
.../src/main/webapp/js/nf/nf-dialog.js | 11 +++---
5 files changed, 60 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/a8724038/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/DropRequestDTO.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/DropRequestDTO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/DropRequestDTO.java
index 93dc7dd..c0b94a1 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/DropRequestDTO.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/DropRequestDTO.java
@@ -36,6 +36,7 @@ public class DropRequestDTO {
private Integer percentCompleted;
private Boolean finished;
+ private String failureReason;
private Integer currentCount;
private Long currentSize;
@@ -125,6 +126,17 @@ public class DropRequestDTO {
}
/**
+ * @return the reason, if any, that this drop request failed
+ */
+ public String getFailureReason() {
+ return failureReason;
+ }
+
+ public void setFailureReason(String failureReason) {
+ this.failureReason = failureReason;
+ }
+
+ /**
* @return the time this request was last updated
*/
@XmlJavaTypeAdapter(TimestampAdapter.class)
http://git-wip-us.apache.org/repos/asf/nifi/blob/a8724038/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
index 0758ce2..d34242d 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
@@ -308,7 +308,10 @@ public final class DtoFactory {
dto.setSubmissionTime(new Date(dropRequest.getRequestSubmissionTime()));
dto.setLastUpdated(new Date(dropRequest.getLastUpdated()));
dto.setState(dropRequest.getState().toString());
- dto.setFinished(DropFlowFileState.COMPLETE.equals(dropRequest.getState()));
+ dto.setFailureReason(dropRequest.getFailureReason());
+ dto.setFinished(DropFlowFileState.COMPLETE.equals(dropRequest.getState())
+ || DropFlowFileState.CANCELED.equals(dropRequest.getState())
+ || DropFlowFileState.FAILURE.equals(dropRequest.getState()));
final QueueSize dropped = dropRequest.getDroppedSize();
dto.setDroppedCount(dropped.getObjectCount());
http://git-wip-us.apache.org/repos/asf/nifi/blob/a8724038/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/dialog.css
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/dialog.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/dialog.css
index 6cb6dba..9f4fefb 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/dialog.css
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/dialog.css
@@ -272,11 +272,16 @@ div.go-to-link {
#drop-request-status-dialog {
display: none;
width: 400px;
- height: 135px;
+ height: 125px;
z-index: 1301;
}
+#drop-request-status-message {
+
+}
+
#drop-request-percent-complete {
+ margin-top: 10px;
width: 378px;
border-radius: 0;
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/a8724038/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js
index ba144ef..3bd9c54 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js
@@ -34,7 +34,6 @@ nf.Actions = (function () {
// configure the drop request status dialog
$('#drop-request-status-dialog').modal({
- headerText: 'Emptying queue',
overlayBackground: false,
handler: {
close: function () {
@@ -44,8 +43,14 @@ nf.Actions = (function () {
// update the progress bar
var label = $('<div class="progress-label"></div>').text('0%');
dropRequestProgressBar.progressbar('value', 0).append(label);
+
+ // clear the current button model
+ $('#drop-request-status-dialog').modal('setButtonModel', []);
}
}
+ }).draggable({
+ containment: 'parent',
+ handle: '.dialog-header'
});
};
@@ -889,8 +894,11 @@ nf.Actions = (function () {
// prompt the user before emptying the queue
nf.Dialog.showYesNoDialog({
- dialogContent: 'Are you sure you want to empty the queue? All data enqueued at the time of the request will be removed from the dataflow.',
+ headerText: 'Empty Queue',
+ dialogContent: 'Are you sure you want to empty this queue? All flowfiles waiting at the time of the request will be removed.',
overlayBackground: false,
+ noText: 'Cancel',
+ yesText: 'Empty',
yesHandler: function () {
// get the connection data
var connection = selection.datum();
@@ -916,7 +924,7 @@ nf.Actions = (function () {
// update the button model of the drop request status dialog
$('#drop-request-status-dialog').modal('setButtonModel', [{
- buttonText: 'Cancel',
+ buttonText: 'Stop',
handler: {
click: function () {
cancelled = true;
@@ -944,9 +952,26 @@ nf.Actions = (function () {
// report the results of this drop request
dropRequest = response.dropRequest;
- // parse the dropped stats to render results
- var tokens = dropRequest.dropped.split(/ \/ /);
- var results = $('<div></div>').text('Successfully removed ' + tokens[0] + ' (' + tokens[1] + ') FlowFiles');
+ // build the results
+ var droppedTokens = dropRequest.dropped.split(/ \/ /);
+ var results = $('<div></div>');
+ $('<span class="label"></span>').text(droppedTokens[0]).appendTo(results);
+ $('<span></span>').text(' FlowFiles (' + droppedTokens[1] + ')').appendTo(results);
+
+ // if the request did not complete, include the original
+ if (dropRequest.percentCompleted < 100) {
+ var originalTokens = dropRequest.original.split(/ \/ /);
+ $('<span class="label"></span>').text(' out of ' + originalTokens[0]).appendTo(results);
+ $('<span></span>').text(' (' + originalTokens[1] + ')').appendTo(results);
+ }
+ $('<span></span>').text(' were removed from the queue.').appendTo(results);
+
+ // if this request failed so the error
+ if (nf.Common.isDefinedAndNotNull(dropRequest.failureReason)) {
+ $('<br/><br/><span></span>').text(dropRequest.failureReason).appendTo(results);
+ }
+
+ // display the results
nf.Dialog.showOkDialog({
dialogContent: results,
overlayBackground: false
http://git-wip-us.apache.org/repos/asf/nifi/blob/a8724038/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/nf-dialog.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/nf-dialog.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/nf-dialog.js
index 2d1183f..13f5cf3 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/nf-dialog.js
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/nf-dialog.js
@@ -47,8 +47,9 @@ $(document).ready(function () {
$('#nf-yes-no-dialog').modal({
handler: {
close: function () {
- // clear the content
+ // clear the content and reset the button model
$('#nf-yes-no-dialog-content').empty();
+ $('#nf-yes-no-dialog').modal('setButtonModel', []);
}
}
}).draggable({
@@ -91,7 +92,9 @@ nf.Dialog = (function () {
options = $.extend({
headerText: '',
dialogContent: '',
- overlayBackgrond: true
+ overlayBackgrond: true,
+ yesText: 'Yes',
+ noText: 'No'
}, options);
// add the content to the prompt
@@ -100,7 +103,7 @@ nf.Dialog = (function () {
// update the button model
$('#nf-yes-no-dialog').modal('setButtonModel', [{
- buttonText: 'Yes',
+ buttonText: options.yesText,
handler: {
click: function () {
// close the dialog
@@ -111,7 +114,7 @@ nf.Dialog = (function () {
}
}
}, {
- buttonText: 'No',
+ buttonText: options.noText,
handler: {
click: function () {
// close the dialog
[25/30] nifi git commit: NIFI-730: - Fixing capitalization to remain
consistent.
Posted by ma...@apache.org.
NIFI-730:
- Fixing capitalization to remain consistent.
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/570202eb
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/570202eb
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/570202eb
Branch: refs/heads/master
Commit: 570202eb3059595394599300e810c22eb801d3cd
Parents: a872403
Author: Matt Gilman <ma...@gmail.com>
Authored: Fri Oct 16 10:47:02 2015 -0400
Committer: Matt Gilman <ma...@gmail.com>
Committed: Fri Oct 16 10:47:02 2015 -0400
----------------------------------------------------------------------
.../org/apache/nifi/controller/queue/DropFlowFileState.java | 6 +++---
.../nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js | 2 +-
2 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/570202eb/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileState.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileState.java b/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileState.java
index 32efcbb..0f258b0 100644
--- a/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileState.java
+++ b/nifi-api/src/main/java/org/apache/nifi/controller/queue/DropFlowFileState.java
@@ -21,11 +21,11 @@ package org.apache.nifi.controller.queue;
*/
public enum DropFlowFileState {
- WAITING_FOR_LOCK("Waiting for Destination Component to complete its action"),
+ WAITING_FOR_LOCK("Waiting for destination component to complete its action"),
DROPPING_FLOWFILES("Dropping FlowFiles from queue"),
FAILURE("Failed"),
- CANCELED("Canceled by User"),
- COMPLETE("Completed Successfully");
+ CANCELED("Canceled by user"),
+ COMPLETE("Completed successfully");
private final String description;
http://git-wip-us.apache.org/repos/asf/nifi/blob/570202eb/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js
index 3bd9c54..e67472a 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js
@@ -895,7 +895,7 @@ nf.Actions = (function () {
// prompt the user before emptying the queue
nf.Dialog.showYesNoDialog({
headerText: 'Empty Queue',
- dialogContent: 'Are you sure you want to empty this queue? All flowfiles waiting at the time of the request will be removed.',
+ dialogContent: 'Are you sure you want to empty this queue? All FlowFiles waiting at the time of the request will be removed.',
overlayBackground: false,
noText: 'Cancel',
yesText: 'Empty',