You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by jo...@apache.org on 2015/08/16 21:58:56 UTC
[40/53] [abbrv] [partial] nifi git commit: NIFI-850 removed nifi
parent, updated nifi pom, moved all nifi subdirs up one level, fixed readme.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/RunNiFi.java
----------------------------------------------------------------------
diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/RunNiFi.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/RunNiFi.java
new file mode 100644
index 0000000..a48a1de
--- /dev/null
+++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/RunNiFi.java
@@ -0,0 +1,1008 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.bootstrap;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.Reader;
+import java.lang.reflect.Field;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.attribute.PosixFilePermission;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ * <p>
+ * The class which bootstraps Apache NiFi. This class looks for the
+ * bootstrap.conf file by looking in the following places (in order):</p>
+ * <ol>
+ * <li>Java System Property named
+ * {@code org.apache.nifi.bootstrap.config.file}</li>
+ * <li>${NIFI_HOME}/./conf/bootstrap.conf, where ${NIFI_HOME} references an
+ * environment variable {@code NIFI_HOME}</li>
+ * <li>./conf/bootstrap.conf, where {@code ./} represents the working
+ * directory.</li>
+ * </ol>
+ *
+ * If the {@code bootstrap.conf} file cannot be found, throws a {@code FileNotFoundException}.
+ */
+public class RunNiFi {
+
+ public static final String DEFAULT_CONFIG_FILE = "./conf/bootstrap.conf";
+ public static final String DEFAULT_NIFI_PROPS_FILE = "./conf/nifi.properties";
+ public static final String DEFAULT_JAVA_CMD = "java";
+
+ public static final String GRACEFUL_SHUTDOWN_PROP = "graceful.shutdown.seconds";
+ public static final String DEFAULT_GRACEFUL_SHUTDOWN_VALUE = "20";
+
+ public static final String RUN_AS_PROP = "run.as";
+
+ public static final int MAX_RESTART_ATTEMPTS = 5;
+ public static final int STARTUP_WAIT_SECONDS = 60;
+
+ public static final String SHUTDOWN_CMD = "SHUTDOWN";
+ public static final String PING_CMD = "PING";
+ public static final String DUMP_CMD = "DUMP";
+
+ private volatile boolean autoRestartNiFi = true;
+ private volatile int ccPort = -1;
+ private volatile long nifiPid = -1L;
+ private volatile String secretKey;
+ private volatile ShutdownHook shutdownHook;
+
+ private final Lock lock = new ReentrantLock();
+ private final Condition startupCondition = lock.newCondition();
+
+ private final File bootstrapConfigFile;
+
+ // used for logging initial info; these will be logged to console by default when the app is started
+ private final Logger cmdLogger = LoggerFactory.getLogger("org.apache.nifi.bootstrap.Command");
+ // used for logging all info. These by default will be written to the log file
+ private final Logger defaultLogger = LoggerFactory.getLogger(RunNiFi.class);
+
+
+ private final ExecutorService loggingExecutor;
+ private volatile Set<Future<?>> loggingFutures = new HashSet<>(2);
+
+ public RunNiFi(final File bootstrapConfigFile, final boolean verbose) {
+ this.bootstrapConfigFile = bootstrapConfigFile;
+
+ loggingExecutor = Executors.newFixedThreadPool(2, new ThreadFactory() {
+ @Override
+ public Thread newThread(final Runnable runnable) {
+ final Thread t = Executors.defaultThreadFactory().newThread(runnable);
+ t.setDaemon(true);
+ t.setName("NiFi logging handler");
+ return t;
+ }
+ });
+ }
+
+ private static void printUsage() {
+ System.out.println("Usage:");
+ System.out.println();
+ System.out.println("java org.apache.nifi.bootstrap.RunNiFi [<-verbose>] <command> [options]");
+ System.out.println();
+ System.out.println("Valid commands include:");
+ System.out.println("");
+ System.out.println("Start : Start a new instance of Apache NiFi");
+ System.out.println("Stop : Stop a running instance of Apache NiFi");
+ System.out.println("Restart : Stop Apache NiFi, if it is running, and then start a new instance");
+ System.out.println("Status : Determine if there is a running instance of Apache NiFi");
+ System.out.println("Dump : Write a Thread Dump to the file specified by [options], or to the log if no file is given");
+ System.out.println("Run : Start a new instance of Apache NiFi and monitor the Process, restarting if the instance dies");
+ System.out.println();
+ }
+
+ private static String[] shift(final String[] orig) {
+ return Arrays.copyOfRange(orig, 1, orig.length);
+ }
+
+ public static void main(String[] args) throws IOException, InterruptedException {
+ if (args.length < 1 || args.length > 3) {
+ printUsage();
+ return;
+ }
+
+ File dumpFile = null;
+ boolean verbose = false;
+ if (args[0].equals("-verbose")) {
+ verbose = true;
+ args = shift(args);
+ }
+
+ final String cmd = args[0];
+ if (cmd.equals("dump")) {
+ if (args.length > 1) {
+ dumpFile = new File(args[1]);
+ } else {
+ dumpFile = null;
+ }
+ }
+
+ switch (cmd.toLowerCase()) {
+ case "start":
+ case "run":
+ case "stop":
+ case "status":
+ case "dump":
+ case "restart":
+ break;
+ default:
+ printUsage();
+ return;
+ }
+
+ String configFilename = System.getProperty("org.apache.nifi.bootstrap.config.file");
+
+ if (configFilename == null) {
+ final String nifiHome = System.getenv("NIFI_HOME");
+ if (nifiHome != null) {
+ final File nifiHomeFile = new File(nifiHome.trim());
+ final File configFile = new File(nifiHomeFile, DEFAULT_CONFIG_FILE);
+ configFilename = configFile.getAbsolutePath();
+ }
+ }
+
+ if (configFilename == null) {
+ configFilename = DEFAULT_CONFIG_FILE;
+ }
+
+ final File configFile = new File(configFilename);
+
+ final RunNiFi runNiFi = new RunNiFi(configFile, verbose);
+
+ switch (cmd.toLowerCase()) {
+ case "start":
+ runNiFi.start();
+ break;
+ case "run":
+ runNiFi.start();
+ break;
+ case "stop":
+ runNiFi.stop();
+ break;
+ case "status":
+ runNiFi.status();
+ break;
+ case "restart":
+ runNiFi.stop();
+ runNiFi.start();
+ break;
+ case "dump":
+ runNiFi.dump(dumpFile);
+ break;
+ }
+ }
+
+ File getStatusFile() {
+ return getStatusFile(defaultLogger);
+ }
+
+ public File getStatusFile(final Logger logger) {
+ final File confDir = bootstrapConfigFile.getParentFile();
+ final File nifiHome = confDir.getParentFile();
+ final File bin = new File(nifiHome, "bin");
+ final File statusFile = new File(bin, "nifi.pid");
+
+ logger.debug("Status File: {}", statusFile);
+
+ return statusFile;
+ }
+
+ private Properties loadProperties(final Logger logger) throws IOException {
+ final Properties props = new Properties();
+ final File statusFile = getStatusFile(logger);
+ if (statusFile == null || !statusFile.exists()) {
+ logger.debug("No status file to load properties from");
+ return props;
+ }
+
+ try (final FileInputStream fis = new FileInputStream(getStatusFile(logger))) {
+ props.load(fis);
+ }
+
+ final Map<Object, Object> modified = new HashMap<>(props);
+ modified.remove("secret.key");
+ logger.debug("Properties: {}", modified);
+
+ return props;
+ }
+
+ private synchronized void saveProperties(final Properties nifiProps, final Logger logger) throws IOException {
+ final File statusFile = getStatusFile(logger);
+ if (statusFile.exists() && !statusFile.delete()) {
+ logger.warn("Failed to delete {}", statusFile);
+ }
+
+ if (!statusFile.createNewFile()) {
+ throw new IOException("Failed to create file " + statusFile);
+ }
+
+ try {
+ final Set<PosixFilePermission> perms = new HashSet<>();
+ perms.add(PosixFilePermission.OWNER_READ);
+ perms.add(PosixFilePermission.OWNER_WRITE);
+ Files.setPosixFilePermissions(statusFile.toPath(), perms);
+ } catch (final Exception e) {
+ logger.warn("Failed to set permissions so that only the owner can read status file {}; "
+ + "this may allows others to have access to the key needed to communicate with NiFi. "
+ + "Permissions should be changed so that only the owner can read this file", statusFile);
+ }
+
+ try (final FileOutputStream fos = new FileOutputStream(statusFile)) {
+ nifiProps.store(fos, null);
+ fos.getFD().sync();
+ }
+
+ logger.debug("Saved Properties {} to {}", new Object[]{nifiProps, statusFile});
+ }
+
+ private boolean isPingSuccessful(final int port, final String secretKey, final Logger logger) {
+ logger.debug("Pinging {}", port);
+
+ try (final Socket socket = new Socket("localhost", port)) {
+ final OutputStream out = socket.getOutputStream();
+ out.write((PING_CMD + " " + secretKey + "\n").getBytes(StandardCharsets.UTF_8));
+ out.flush();
+
+ logger.debug("Sent PING command");
+ socket.setSoTimeout(5000);
+ final InputStream in = socket.getInputStream();
+ final BufferedReader reader = new BufferedReader(new InputStreamReader(in));
+ final String response = reader.readLine();
+ logger.debug("PING response: {}", response);
+ out.close();
+ reader.close();
+
+ return PING_CMD.equals(response);
+ } catch (final IOException ioe) {
+ return false;
+ }
+ }
+
+ private Integer getCurrentPort(final Logger logger) throws IOException {
+ final Properties props = loadProperties(logger);
+ final String portVal = props.getProperty("port");
+ if (portVal == null) {
+ logger.debug("No Port found in status file");
+ return null;
+ } else {
+ logger.debug("Port defined in status file: {}", portVal);
+ }
+
+ final int port = Integer.parseInt(portVal);
+ final boolean success = isPingSuccessful(port, props.getProperty("secret.key"), logger);
+ if (success) {
+ logger.debug("Successful PING on port {}", port);
+ return port;
+ }
+
+ final String pid = props.getProperty("pid");
+ logger.debug("PID in status file is {}", pid);
+ if (pid != null) {
+ final boolean procRunning = isProcessRunning(pid, logger);
+ if (procRunning) {
+ return port;
+ } else {
+ return null;
+ }
+ }
+
+ return null;
+ }
+
+ private boolean isProcessRunning(final String pid, final Logger logger) {
+ try {
+ // We use the "ps" command to check if the process is still running.
+ final ProcessBuilder builder = new ProcessBuilder();
+
+ builder.command("ps", "-p", pid);
+ final Process proc = builder.start();
+
+ // Look for the pid in the output of the 'ps' command.
+ boolean running = false;
+ String line;
+ try (final InputStream in = proc.getInputStream();
+ final Reader streamReader = new InputStreamReader(in);
+ final BufferedReader reader = new BufferedReader(streamReader)) {
+
+ while ((line = reader.readLine()) != null) {
+ if (line.trim().startsWith(pid)) {
+ running = true;
+ }
+ }
+ }
+
+ // If output of the ps command had our PID, the process is running.
+ if (running) {
+ logger.debug("Process with PID {} is running", pid);
+ } else {
+ logger.debug("Process with PID {} is not running", pid);
+ }
+
+ return running;
+ } catch (final IOException ioe) {
+ System.err.println("Failed to determine if Process " + pid + " is running; assuming that it is not");
+ return false;
+ }
+ }
+
+ private Status getStatus(final Logger logger) {
+ final Properties props;
+ try {
+ props = loadProperties(logger);
+ } catch (final IOException ioe) {
+ return new Status(null, null, false, false);
+ }
+
+ if (props == null) {
+ return new Status(null, null, false, false);
+ }
+
+ final String portValue = props.getProperty("port");
+ final String pid = props.getProperty("pid");
+ final String secretKey = props.getProperty("secret.key");
+
+ if (portValue == null && pid == null) {
+ return new Status(null, null, false, false);
+ }
+
+ Integer port = null;
+ boolean pingSuccess = false;
+ if (portValue != null) {
+ try {
+ port = Integer.parseInt(portValue);
+ pingSuccess = isPingSuccessful(port, secretKey, logger);
+ } catch (final NumberFormatException nfe) {
+ return new Status(null, null, false, false);
+ }
+ }
+
+ if (pingSuccess) {
+ return new Status(port, pid, true, true);
+ }
+
+ final boolean alive = (pid == null) ? false : isProcessRunning(pid, logger);
+ return new Status(port, pid, pingSuccess, alive);
+ }
+
+ public void status() throws IOException {
+ final Logger logger = cmdLogger;
+ final Status status = getStatus(logger);
+ if (status.isRespondingToPing()) {
+ logger.info("Apache NiFi is currently running, listening to Bootstrap on port {}, PID={}",
+ new Object[]{status.getPort(), status.getPid() == null ? "unknkown" : status.getPid()});
+ return;
+ }
+
+ if (status.isProcessRunning()) {
+ logger.info("Apache NiFi is running at PID {} but is not responding to ping requests", status.getPid());
+ return;
+ }
+
+ if (status.getPort() == null) {
+ logger.info("Apache NiFi is not running");
+ return;
+ }
+
+ if (status.getPid() == null) {
+ logger.info("Apache NiFi is not responding to Ping requests. The process may have died or may be hung");
+ } else {
+ logger.info("Apache NiFi is not running");
+ }
+ }
+
+ /**
+ * Writes a NiFi thread dump to the given file; if file is null, logs at
+ * INFO level instead.
+ *
+ * @param dumpFile the file to write the dump content to
+ * @throws IOException if any issues occur while writing the dump file
+ */
+ public void dump(final File dumpFile) throws IOException {
+ final Logger logger = defaultLogger; // dump to bootstrap log file by default
+ final Integer port = getCurrentPort(logger);
+ if (port == null) {
+ logger.info("Apache NiFi is not currently running");
+ return;
+ }
+
+ final Properties nifiProps = loadProperties(logger);
+ final String secretKey = nifiProps.getProperty("secret.key");
+
+ final StringBuilder sb = new StringBuilder();
+ try (final Socket socket = new Socket()) {
+ logger.debug("Connecting to NiFi instance");
+ socket.setSoTimeout(60000);
+ socket.connect(new InetSocketAddress("localhost", port));
+ logger.debug("Established connection to NiFi instance.");
+ socket.setSoTimeout(60000);
+
+ logger.debug("Sending DUMP Command to port {}", port);
+ final OutputStream out = socket.getOutputStream();
+ out.write((DUMP_CMD + " " + secretKey + "\n").getBytes(StandardCharsets.UTF_8));
+ out.flush();
+
+ final InputStream in = socket.getInputStream();
+ try (final BufferedReader reader = new BufferedReader(new InputStreamReader(in))) {
+ String line;
+ while ((line = reader.readLine()) != null) {
+ sb.append(line).append("\n");
+ }
+ }
+ }
+
+ final String dump = sb.toString();
+ if (dumpFile == null) {
+ logger.info(dump);
+ } else {
+ try (final FileOutputStream fos = new FileOutputStream(dumpFile)) {
+ fos.write(dump.getBytes(StandardCharsets.UTF_8));
+ }
+ // we want to log to the console (by default) that we wrote the thread dump to the specified file
+ cmdLogger.info("Successfully wrote thread dump to {}", dumpFile.getAbsolutePath());
+ }
+ }
+
+ public void stop() throws IOException {
+ final Logger logger = cmdLogger;
+ final Integer port = getCurrentPort(logger);
+ if (port == null) {
+ logger.info("Apache NiFi is not currently running");
+ return;
+ }
+
+ final Properties nifiProps = loadProperties(logger);
+ final String secretKey = nifiProps.getProperty("secret.key");
+
+ final File statusFile = getStatusFile(logger);
+ if (statusFile.exists() && !statusFile.delete()) {
+ logger.error("Failed to delete status file {}; this file should be cleaned up manually", statusFile);
+ }
+
+ try (final Socket socket = new Socket()) {
+ logger.debug("Connecting to NiFi instance");
+ socket.setSoTimeout(60000);
+ socket.connect(new InetSocketAddress("localhost", port));
+ logger.debug("Established connection to NiFi instance.");
+ socket.setSoTimeout(60000);
+
+ logger.debug("Sending SHUTDOWN Command to port {}", port);
+ final OutputStream out = socket.getOutputStream();
+ out.write((SHUTDOWN_CMD + " " + secretKey + "\n").getBytes(StandardCharsets.UTF_8));
+ out.flush();
+ socket.shutdownOutput();
+
+ final InputStream in = socket.getInputStream();
+ int lastChar;
+ final StringBuilder sb = new StringBuilder();
+ while ((lastChar = in.read()) > -1) {
+ sb.append((char) lastChar);
+ }
+ final String response = sb.toString().trim();
+
+ logger.debug("Received response to SHUTDOWN command: {}", response);
+
+ if (SHUTDOWN_CMD.equals(response)) {
+ logger.info("Apache NiFi has accepted the Shutdown Command and is shutting down now");
+
+ final String pid = nifiProps.getProperty("pid");
+ if (pid != null) {
+ final Properties bootstrapProperties = new Properties();
+ try (final FileInputStream fis = new FileInputStream(bootstrapConfigFile)) {
+ bootstrapProperties.load(fis);
+ }
+
+ String gracefulShutdown = bootstrapProperties.getProperty(GRACEFUL_SHUTDOWN_PROP, DEFAULT_GRACEFUL_SHUTDOWN_VALUE);
+ int gracefulShutdownSeconds;
+ try {
+ gracefulShutdownSeconds = Integer.parseInt(gracefulShutdown);
+ } catch (final NumberFormatException nfe) {
+ gracefulShutdownSeconds = Integer.parseInt(DEFAULT_GRACEFUL_SHUTDOWN_VALUE);
+ }
+
+ final long startWait = System.nanoTime();
+ while (isProcessRunning(pid, logger)) {
+ logger.info("Waiting for Apache NiFi to finish shutting down...");
+ final long waitNanos = System.nanoTime() - startWait;
+ final long waitSeconds = TimeUnit.NANOSECONDS.toSeconds(waitNanos);
+ if (waitSeconds >= gracefulShutdownSeconds && gracefulShutdownSeconds > 0) {
+ if (isProcessRunning(pid, logger)) {
+ logger.warn("NiFi has not finished shutting down after {} seconds. Killing process.", gracefulShutdownSeconds);
+ try {
+ killProcessTree(pid, logger);
+ } catch (final IOException ioe) {
+ logger.error("Failed to kill Process with PID {}", pid);
+ }
+ }
+ break;
+ } else {
+ try {
+ Thread.sleep(2000L);
+ } catch (final InterruptedException ie) {
+ }
+ }
+ }
+
+ logger.info("NiFi has finished shutting down.");
+ }
+ } else {
+ logger.error("When sending SHUTDOWN command to NiFi, got unexpected response {}", response);
+ }
+ } catch (final IOException ioe) {
+ logger.error("Failed to send shutdown command to port {} due to {}", new Object[]{port, ioe.toString(), ioe});
+ }
+ }
+
+ private static List<String> getChildProcesses(final String ppid) throws IOException {
+ final Process proc = Runtime.getRuntime().exec(new String[]{"ps", "-o", "pid", "--no-headers", "--ppid", ppid});
+ final List<String> childPids = new ArrayList<>();
+ try (final InputStream in = proc.getInputStream();
+ final BufferedReader reader = new BufferedReader(new InputStreamReader(in))) {
+
+ String line;
+ while ((line = reader.readLine()) != null) {
+ childPids.add(line.trim());
+ }
+ }
+
+ return childPids;
+ }
+
+ private void killProcessTree(final String pid, final Logger logger) throws IOException {
+ logger.debug("Killing Process Tree for PID {}", pid);
+
+ final List<String> children = getChildProcesses(pid);
+ logger.debug("Children of PID {}: {}", new Object[]{pid, children});
+
+ for (final String childPid : children) {
+ killProcessTree(childPid, logger);
+ }
+
+ Runtime.getRuntime().exec(new String[]{"kill", "-9", pid});
+ }
+
+ public static boolean isAlive(final Process process) {
+ try {
+ process.exitValue();
+ return false;
+ } catch (final IllegalStateException | IllegalThreadStateException itse) {
+ return true;
+ }
+ }
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public void start() throws IOException, InterruptedException {
+ final Integer port = getCurrentPort(cmdLogger);
+ if (port != null) {
+ cmdLogger.info("Apache NiFi is already running, listening to Bootstrap on port " + port);
+ return;
+ }
+
+ final ProcessBuilder builder = new ProcessBuilder();
+
+ if (!bootstrapConfigFile.exists()) {
+ throw new FileNotFoundException(bootstrapConfigFile.getAbsolutePath());
+ }
+
+ final Properties properties = new Properties();
+ try (final FileInputStream fis = new FileInputStream(bootstrapConfigFile)) {
+ properties.load(fis);
+ }
+
+ final Map<String, String> props = new HashMap<>();
+ props.putAll((Map) properties);
+
+ final String specifiedWorkingDir = props.get("working.dir");
+ if (specifiedWorkingDir != null) {
+ builder.directory(new File(specifiedWorkingDir));
+ }
+
+ final File bootstrapConfigAbsoluteFile = bootstrapConfigFile.getAbsoluteFile();
+ final File binDir = bootstrapConfigAbsoluteFile.getParentFile();
+ final File workingDir = binDir.getParentFile();
+
+ if (specifiedWorkingDir == null) {
+ builder.directory(workingDir);
+ }
+
+ final String libFilename = replaceNull(props.get("lib.dir"), "./lib").trim();
+ File libDir = getFile(libFilename, workingDir);
+
+ final String confFilename = replaceNull(props.get("conf.dir"), "./conf").trim();
+ File confDir = getFile(confFilename, workingDir);
+
+ String nifiPropsFilename = props.get("props.file");
+ if (nifiPropsFilename == null) {
+ if (confDir.exists()) {
+ nifiPropsFilename = new File(confDir, "nifi.properties").getAbsolutePath();
+ } else {
+ nifiPropsFilename = DEFAULT_CONFIG_FILE;
+ }
+ }
+
+ nifiPropsFilename = nifiPropsFilename.trim();
+
+ final List<String> javaAdditionalArgs = new ArrayList<>();
+ for (final Map.Entry<String, String> entry : props.entrySet()) {
+ final String key = entry.getKey();
+ final String value = entry.getValue();
+
+ if (key.startsWith("java.arg")) {
+ javaAdditionalArgs.add(value);
+ }
+ }
+
+ final File[] libFiles = libDir.listFiles(new FilenameFilter() {
+ @Override
+ public boolean accept(final File dir, final String filename) {
+ return filename.toLowerCase().endsWith(".jar");
+ }
+ });
+
+ if (libFiles == null || libFiles.length == 0) {
+ throw new RuntimeException("Could not find lib directory at " + libDir.getAbsolutePath());
+ }
+
+ final File[] confFiles = confDir.listFiles();
+ if (confFiles == null || confFiles.length == 0) {
+ throw new RuntimeException("Could not find conf directory at " + confDir.getAbsolutePath());
+ }
+
+ final List<String> cpFiles = new ArrayList<>(confFiles.length + libFiles.length);
+ cpFiles.add(confDir.getAbsolutePath());
+ for (final File file : libFiles) {
+ cpFiles.add(file.getAbsolutePath());
+ }
+
+ final StringBuilder classPathBuilder = new StringBuilder();
+ for (int i = 0; i < cpFiles.size(); i++) {
+ final String filename = cpFiles.get(i);
+ classPathBuilder.append(filename);
+ if (i < cpFiles.size() - 1) {
+ classPathBuilder.append(File.pathSeparatorChar);
+ }
+ }
+
+ final String classPath = classPathBuilder.toString();
+ String javaCmd = props.get("java");
+ if (javaCmd == null) {
+ javaCmd = DEFAULT_JAVA_CMD;
+ }
+ if (javaCmd.equals(DEFAULT_JAVA_CMD)) {
+ String javaHome = System.getenv("JAVA_HOME");
+ if (javaHome != null) {
+ String fileExtension = isWindows() ? ".exe" : "";
+ File javaFile = new File(javaHome + File.separatorChar + "bin"
+ + File.separatorChar + "java" + fileExtension);
+ if (javaFile.exists() && javaFile.canExecute()) {
+ javaCmd = javaFile.getAbsolutePath();
+ }
+ }
+ }
+
+ final NiFiListener listener = new NiFiListener();
+ final int listenPort = listener.start(this);
+
+ final List<String> cmd = new ArrayList<>();
+
+ cmd.add(javaCmd);
+ cmd.add("-classpath");
+ cmd.add(classPath);
+ cmd.addAll(javaAdditionalArgs);
+ cmd.add("-Dnifi.properties.file.path=" + nifiPropsFilename);
+ cmd.add("-Dnifi.bootstrap.listen.port=" + listenPort);
+ cmd.add("-Dapp=NiFi");
+ cmd.add("org.apache.nifi.NiFi");
+
+ builder.command(cmd);
+
+ final StringBuilder cmdBuilder = new StringBuilder();
+ for (final String s : cmd) {
+ cmdBuilder.append(s).append(" ");
+ }
+
+ cmdLogger.info("Starting Apache NiFi...");
+ cmdLogger.info("Working Directory: {}", workingDir.getAbsolutePath());
+ cmdLogger.info("Command: {}", cmdBuilder.toString());
+
+ String gracefulShutdown = props.get(GRACEFUL_SHUTDOWN_PROP);
+ if (gracefulShutdown == null) {
+ gracefulShutdown = DEFAULT_GRACEFUL_SHUTDOWN_VALUE;
+ }
+
+ final int gracefulShutdownSeconds;
+ try {
+ gracefulShutdownSeconds = Integer.parseInt(gracefulShutdown);
+ } catch (final NumberFormatException nfe) {
+ throw new NumberFormatException("The '" + GRACEFUL_SHUTDOWN_PROP + "' property in Bootstrap Config File "
+ + bootstrapConfigAbsoluteFile.getAbsolutePath() + " has an invalid value. Must be a non-negative integer");
+ }
+
+ if (gracefulShutdownSeconds < 0) {
+ throw new NumberFormatException("The '" + GRACEFUL_SHUTDOWN_PROP + "' property in Bootstrap Config File "
+ + bootstrapConfigAbsoluteFile.getAbsolutePath() + " has an invalid value. Must be a non-negative integer");
+ }
+
+ Process process = builder.start();
+ handleLogging(process);
+ Long pid = getPid(process, cmdLogger);
+ if (pid != null) {
+ nifiPid = pid;
+ final Properties nifiProps = new Properties();
+ nifiProps.setProperty("pid", String.valueOf(nifiPid));
+ saveProperties(nifiProps, cmdLogger);
+ }
+
+ shutdownHook = new ShutdownHook(process, this, secretKey, gracefulShutdownSeconds, loggingExecutor);
+ final Runtime runtime = Runtime.getRuntime();
+ runtime.addShutdownHook(shutdownHook);
+
+ while (true) {
+ final boolean alive = isAlive(process);
+
+ if (alive) {
+ try {
+ Thread.sleep(1000L);
+ } catch (final InterruptedException ie) {
+ }
+ } else {
+ try {
+ runtime.removeShutdownHook(shutdownHook);
+ } catch (final IllegalStateException ise) {
+ // happens when already shutting down
+ }
+
+ if (autoRestartNiFi) {
+ final File statusFile = getStatusFile(defaultLogger);
+ if (!statusFile.exists()) {
+ defaultLogger.debug("Status File no longer exists. Will not restart NiFi");
+ return;
+ }
+
+ defaultLogger.warn("Apache NiFi appears to have died. Restarting...");
+ process = builder.start();
+ handleLogging(process);
+
+ pid = getPid(process, defaultLogger);
+ if (pid != null) {
+ nifiPid = pid;
+ final Properties nifiProps = new Properties();
+ nifiProps.setProperty("pid", String.valueOf(nifiPid));
+ saveProperties(nifiProps, defaultLogger);
+ }
+
+ shutdownHook = new ShutdownHook(process, this, secretKey, gracefulShutdownSeconds, loggingExecutor);
+ runtime.addShutdownHook(shutdownHook);
+
+ final boolean started = waitForStart();
+
+ if (started) {
+ defaultLogger.info("Successfully started Apache NiFi{}", (pid == null ? "" : " with PID " + pid));
+ } else {
+ defaultLogger.error("Apache NiFi does not appear to have started");
+ }
+ } else {
+ return;
+ }
+ }
+ }
+ }
+
+ private void handleLogging(final Process process) {
+ final Set<Future<?>> existingFutures = loggingFutures;
+ if (existingFutures != null) {
+ for (final Future<?> future : existingFutures) {
+ future.cancel(false);
+ }
+ }
+
+ final Future<?> stdOutFuture = loggingExecutor.submit(new Runnable() {
+ @Override
+ public void run() {
+ final Logger stdOutLogger = LoggerFactory.getLogger("org.apache.nifi.StdOut");
+ final InputStream in = process.getInputStream();
+ try (final BufferedReader reader = new BufferedReader(new InputStreamReader(in))) {
+ String line;
+ while ((line = reader.readLine()) != null) {
+ stdOutLogger.info(line);
+ }
+ } catch (IOException e) {
+ defaultLogger.error("Failed to read from NiFi's Standard Out stream", e);
+ }
+ }
+ });
+
+ final Future<?> stdErrFuture = loggingExecutor.submit(new Runnable() {
+ @Override
+ public void run() {
+ final Logger stdErrLogger = LoggerFactory.getLogger("org.apache.nifi.StdErr");
+ final InputStream in = process.getErrorStream();
+ try (final BufferedReader reader = new BufferedReader(new InputStreamReader(in))) {
+ String line;
+ while ((line = reader.readLine()) != null) {
+ stdErrLogger.error(line);
+ }
+ } catch (IOException e) {
+ defaultLogger.error("Failed to read from NiFi's Standard Error stream", e);
+ }
+ }
+ });
+
+ final Set<Future<?>> futures = new HashSet<>();
+ futures.add(stdOutFuture);
+ futures.add(stdErrFuture);
+ this.loggingFutures = futures;
+ }
+
+ private Long getPid(final Process process, final Logger logger) {
+ try {
+ final Class<?> procClass = process.getClass();
+ final Field pidField = procClass.getDeclaredField("pid");
+ pidField.setAccessible(true);
+ final Object pidObject = pidField.get(process);
+
+ logger.debug("PID Object = {}", pidObject);
+
+ if (pidObject instanceof Number) {
+ return ((Number) pidObject).longValue();
+ }
+ return null;
+ } catch (final IllegalAccessException | NoSuchFieldException nsfe) {
+ logger.debug("Could not find PID for child process due to {}", nsfe);
+ return null;
+ }
+ }
+
+ private boolean isWindows() {
+ final String osName = System.getProperty("os.name");
+ return osName != null && osName.toLowerCase().contains("win");
+ }
+
+ private boolean waitForStart() {
+ lock.lock();
+ try {
+ final long startTime = System.nanoTime();
+
+ while (ccPort < 1) {
+ try {
+ startupCondition.await(1, TimeUnit.SECONDS);
+ } catch (final InterruptedException ie) {
+ return false;
+ }
+
+ final long waitNanos = System.nanoTime() - startTime;
+ final long waitSeconds = TimeUnit.NANOSECONDS.toSeconds(waitNanos);
+ if (waitSeconds > STARTUP_WAIT_SECONDS) {
+ return false;
+ }
+ }
+ } finally {
+ lock.unlock();
+ }
+ return true;
+ }
+
+ private File getFile(final String filename, final File workingDir) {
+ File file = new File(filename);
+ if (!file.isAbsolute()) {
+ file = new File(workingDir, filename);
+ }
+
+ return file;
+ }
+
+ private String replaceNull(final String value, final String replacement) {
+ return (value == null) ? replacement : value;
+ }
+
+ void setAutoRestartNiFi(final boolean restart) {
+ this.autoRestartNiFi = restart;
+ }
+
+ void setNiFiCommandControlPort(final int port, final String secretKey) {
+ this.ccPort = port;
+ this.secretKey = secretKey;
+
+ if (shutdownHook != null) {
+ shutdownHook.setSecretKey(secretKey);
+ }
+
+ final File statusFile = getStatusFile(defaultLogger);
+
+ final Properties nifiProps = new Properties();
+ if (nifiPid != -1) {
+ nifiProps.setProperty("pid", String.valueOf(nifiPid));
+ }
+ nifiProps.setProperty("port", String.valueOf(ccPort));
+ nifiProps.setProperty("secret.key", secretKey);
+
+ try {
+ saveProperties(nifiProps, defaultLogger);
+ } catch (final IOException ioe) {
+ defaultLogger.warn("Apache NiFi has started but failed to persist NiFi Port information to {} due to {}", new Object[]{statusFile.getAbsolutePath(), ioe});
+ }
+
+ defaultLogger.info("Apache NiFi now running and listening for Bootstrap requests on port {}", port);
+ }
+
+ int getNiFiCommandControlPort() {
+ return this.ccPort;
+ }
+
+ private static class Status {
+
+ private final Integer port;
+ private final String pid;
+
+ private final Boolean respondingToPing;
+ private final Boolean processRunning;
+
+ public Status(final Integer port, final String pid, final Boolean respondingToPing, final Boolean processRunning) {
+ this.port = port;
+ this.pid = pid;
+ this.respondingToPing = respondingToPing;
+ this.processRunning = processRunning;
+ }
+
+ public String getPid() {
+ return pid;
+ }
+
+ public Integer getPort() {
+ return port;
+ }
+
+ public boolean isRespondingToPing() {
+ return Boolean.TRUE.equals(respondingToPing);
+ }
+
+ public boolean isProcessRunning() {
+ return Boolean.TRUE.equals(processRunning);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/ShutdownHook.java
----------------------------------------------------------------------
diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/ShutdownHook.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/ShutdownHook.java
new file mode 100644
index 0000000..a594f60
--- /dev/null
+++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/ShutdownHook.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.bootstrap;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.Socket;
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+
+public class ShutdownHook extends Thread {
+
+ private final Process nifiProcess;
+ private final RunNiFi runner;
+ private final int gracefulShutdownSeconds;
+ private final ExecutorService executor;
+
+ private volatile String secretKey;
+
+ public ShutdownHook(final Process nifiProcess, final RunNiFi runner, final String secretKey, final int gracefulShutdownSeconds, final ExecutorService executor) {
+ this.nifiProcess = nifiProcess;
+ this.runner = runner;
+ this.secretKey = secretKey;
+ this.gracefulShutdownSeconds = gracefulShutdownSeconds;
+ this.executor = executor;
+ }
+
+ void setSecretKey(final String secretKey) {
+ this.secretKey = secretKey;
+ }
+
+ @Override
+ public void run() {
+ executor.shutdown();
+ runner.setAutoRestartNiFi(false);
+ final int ccPort = runner.getNiFiCommandControlPort();
+ if (ccPort > 0) {
+ System.out.println("Initiating Shutdown of NiFi...");
+
+ try {
+ final Socket socket = new Socket("localhost", ccPort);
+ final OutputStream out = socket.getOutputStream();
+ out.write(("SHUTDOWN " + secretKey + "\n").getBytes(StandardCharsets.UTF_8));
+ out.flush();
+
+ socket.close();
+ } catch (final IOException ioe) {
+ System.out.println("Failed to Shutdown NiFi due to " + ioe);
+ }
+ }
+
+ System.out.println("Waiting for Apache NiFi to finish shutting down...");
+ final long startWait = System.nanoTime();
+ while (RunNiFi.isAlive(nifiProcess)) {
+ final long waitNanos = System.nanoTime() - startWait;
+ final long waitSeconds = TimeUnit.NANOSECONDS.toSeconds(waitNanos);
+ if (waitSeconds >= gracefulShutdownSeconds && gracefulShutdownSeconds > 0) {
+ if (RunNiFi.isAlive(nifiProcess)) {
+ System.out.println("NiFi has not finished shutting down after " + gracefulShutdownSeconds + " seconds. Killing process.");
+ nifiProcess.destroy();
+ }
+ break;
+ } else {
+ try {
+ Thread.sleep(1000L);
+ } catch (final InterruptedException ie) {
+ }
+ }
+ }
+
+ final File statusFile = runner.getStatusFile();
+ if (!statusFile.delete()) {
+ System.err.println("Failed to delete status file " + statusFile.getAbsolutePath() + "; this file should be cleaned up manually");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/exception/InvalidCommandException.java
----------------------------------------------------------------------
diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/exception/InvalidCommandException.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/exception/InvalidCommandException.java
new file mode 100644
index 0000000..52e36b9
--- /dev/null
+++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/exception/InvalidCommandException.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.bootstrap.exception;
+
+public class InvalidCommandException extends Exception {
+
+ private static final long serialVersionUID = 1L;
+
+ public InvalidCommandException() {
+ super();
+ }
+
+ public InvalidCommandException(final String message) {
+ super(message);
+ }
+
+ public InvalidCommandException(final Throwable t) {
+ super(t);
+ }
+
+ public InvalidCommandException(final String message, final Throwable t) {
+ super(message, t);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/util/LimitingInputStream.java
----------------------------------------------------------------------
diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/util/LimitingInputStream.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/util/LimitingInputStream.java
new file mode 100644
index 0000000..2149342
--- /dev/null
+++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/util/LimitingInputStream.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.bootstrap.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+public class LimitingInputStream extends InputStream {
+
+ private final InputStream in;
+ private final long limit;
+ private long bytesRead = 0;
+
+ public LimitingInputStream(final InputStream in, final long limit) {
+ this.in = in;
+ this.limit = limit;
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (bytesRead >= limit) {
+ return -1;
+ }
+
+ final int val = in.read();
+ if (val > -1) {
+ bytesRead++;
+ }
+ return val;
+ }
+
+ @Override
+ public int read(final byte[] b) throws IOException {
+ if (bytesRead >= limit) {
+ return -1;
+ }
+
+ final int maxToRead = (int) Math.min(b.length, limit - bytesRead);
+
+ final int val = in.read(b, 0, maxToRead);
+ if (val > 0) {
+ bytesRead += val;
+ }
+ return val;
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ if (bytesRead >= limit) {
+ return -1;
+ }
+
+ final int maxToRead = (int) Math.min(len, limit - bytesRead);
+
+ final int val = in.read(b, off, maxToRead);
+ if (val > 0) {
+ bytesRead += val;
+ }
+ return val;
+ }
+
+ @Override
+ public long skip(final long n) throws IOException {
+ final long skipped = in.skip(Math.min(n, limit - bytesRead));
+ bytesRead += skipped;
+ return skipped;
+ }
+
+ @Override
+ public int available() throws IOException {
+ return in.available();
+ }
+
+ @Override
+ public void close() throws IOException {
+ in.close();
+ }
+
+ @Override
+ public void mark(int readlimit) {
+ in.mark(readlimit);
+ }
+
+ @Override
+ public boolean markSupported() {
+ return in.markSupported();
+ }
+
+ @Override
+ public void reset() throws IOException {
+ in.reset();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-data-provenance-utils/.gitignore
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-data-provenance-utils/.gitignore b/nifi-commons/nifi-data-provenance-utils/.gitignore
new file mode 100755
index 0000000..19f2e00
--- /dev/null
+++ b/nifi-commons/nifi-data-provenance-utils/.gitignore
@@ -0,0 +1,2 @@
+/target
+/target
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-data-provenance-utils/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-data-provenance-utils/pom.xml b/nifi-commons/nifi-data-provenance-utils/pom.xml
new file mode 100644
index 0000000..5481cd2
--- /dev/null
+++ b/nifi-commons/nifi-data-provenance-utils/pom.xml
@@ -0,0 +1,34 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-commons</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>nifi-data-provenance-utils</artifactId>
+ <packaging>jar</packaging>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-utils</artifactId>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncLineageSubmission.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncLineageSubmission.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncLineageSubmission.java
new file mode 100644
index 0000000..4a52a89
--- /dev/null
+++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncLineageSubmission.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import java.util.Collection;
+import java.util.Date;
+import java.util.UUID;
+
+import org.apache.nifi.provenance.lineage.ComputeLineageSubmission;
+import org.apache.nifi.provenance.lineage.LineageComputationType;
+
+/**
+ *
+ */
+public class AsyncLineageSubmission implements ComputeLineageSubmission {
+
+ private final String lineageIdentifier = UUID.randomUUID().toString();
+ private final Date submissionTime = new Date();
+
+ private final LineageComputationType computationType;
+ private final Long eventId;
+ private final Collection<String> lineageFlowFileUuids;
+
+ private volatile boolean canceled = false;
+
+ private final StandardLineageResult result;
+
+ public AsyncLineageSubmission(final LineageComputationType computationType, final Long eventId, final Collection<String> lineageFlowFileUuids, final int numSteps) {
+ this.computationType = computationType;
+ this.eventId = eventId;
+ this.lineageFlowFileUuids = lineageFlowFileUuids;
+ this.result = new StandardLineageResult(numSteps, lineageFlowFileUuids);
+ }
+
+ @Override
+ public StandardLineageResult getResult() {
+ return result;
+ }
+
+ @Override
+ public Date getSubmissionTime() {
+ return submissionTime;
+ }
+
+ @Override
+ public String getLineageIdentifier() {
+ return lineageIdentifier;
+ }
+
+ @Override
+ public void cancel() {
+ this.canceled = true;
+ }
+
+ @Override
+ public boolean isCanceled() {
+ return canceled;
+ }
+
+ @Override
+ public LineageComputationType getLineageComputationType() {
+ return computationType;
+ }
+
+ @Override
+ public Long getExpandedEventId() {
+ return eventId;
+ }
+
+ @Override
+ public Collection<String> getLineageFlowFileUuids() {
+ return lineageFlowFileUuids;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncQuerySubmission.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncQuerySubmission.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncQuerySubmission.java
new file mode 100644
index 0000000..00c6170
--- /dev/null
+++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncQuerySubmission.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import java.util.Date;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.nifi.provenance.search.Query;
+import org.apache.nifi.provenance.search.QuerySubmission;
+
+public class AsyncQuerySubmission implements QuerySubmission {
+
+ public static final int TTL = (int) TimeUnit.MILLISECONDS.convert(60, TimeUnit.SECONDS);
+
+ private final Date submissionTime = new Date();
+ private final Query query;
+
+ private volatile boolean canceled = false;
+ private final StandardQueryResult queryResult;
+
+ /**
+ * Constructs an AsyncQuerySubmission with the given query and the given
+ * number of steps, indicating how many results must be added to this
+ * AsyncQuerySubmission before it is considered finished
+ *
+ * @param query the query to execute
+ * @param numSteps how many steps to include
+ */
+ public AsyncQuerySubmission(final Query query, final int numSteps) {
+ this.query = query;
+ queryResult = new StandardQueryResult(query, numSteps);
+ }
+
+ @Override
+ public Date getSubmissionTime() {
+ return submissionTime;
+ }
+
+ @Override
+ public String getQueryIdentifier() {
+ return query.getIdentifier();
+ }
+
+ @Override
+ public void cancel() {
+ this.canceled = true;
+ queryResult.cancel();
+ }
+
+ @Override
+ public boolean isCanceled() {
+ return canceled;
+ }
+
+ @Override
+ public Query getQuery() {
+ return query;
+ }
+
+ @Override
+ public StandardQueryResult getResult() {
+ return queryResult;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/NamedSearchableField.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/NamedSearchableField.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/NamedSearchableField.java
new file mode 100644
index 0000000..dc2903f
--- /dev/null
+++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/NamedSearchableField.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import org.apache.nifi.provenance.search.SearchableField;
+import org.apache.nifi.provenance.search.SearchableFieldType;
+
+import static java.util.Objects.requireNonNull;
+
+/**
+ *
+ */
+public class NamedSearchableField implements SearchableField {
+
+ private final String identifier;
+ private final String searchableName;
+ private final SearchableFieldType fieldType;
+ private final String friendlyName;
+ private final boolean attribute;
+
+ NamedSearchableField(final String identifier, final String searchableName, final String friendlyName, final boolean attribute) {
+ this(identifier, searchableName, friendlyName, attribute, SearchableFieldType.STRING);
+ }
+
+ NamedSearchableField(final String identifier, final String searchableName, final String friendlyName, final boolean attribute, final SearchableFieldType fieldType) {
+ this.identifier = requireNonNull(identifier);
+ this.searchableName = requireNonNull(searchableName);
+ this.friendlyName = requireNonNull(friendlyName);
+ this.attribute = requireNonNull(attribute);
+ this.fieldType = requireNonNull(fieldType);
+ }
+
+ @Override
+ public String getIdentifier() {
+ return identifier;
+ }
+
+ @Override
+ public String getSearchableFieldName() {
+ return searchableName;
+ }
+
+ @Override
+ public String getFriendlyName() {
+ return friendlyName;
+ }
+
+ @Override
+ public boolean isAttribute() {
+ return attribute;
+ }
+
+ @Override
+ public SearchableFieldType getFieldType() {
+ return fieldType;
+ }
+
+ @Override
+ public String toString() {
+ return friendlyName;
+ }
+
+ @Override
+ public int hashCode() {
+ return 298347 + searchableName.hashCode() + (attribute ? 1 : 0);
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (obj == null) {
+ return false;
+ }
+
+ if (!(obj instanceof SearchableField)) {
+ return false;
+ }
+
+ final SearchableField other = (SearchableField) obj;
+ return (this.searchableName.equals(other.getSearchableFieldName()) && attribute == other.isAttribute());
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFieldParser.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFieldParser.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFieldParser.java
new file mode 100644
index 0000000..6a934b1
--- /dev/null
+++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFieldParser.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.nifi.provenance.search.SearchableField;
+
+public class SearchableFieldParser {
+
+ public static List<SearchableField> extractSearchableFields(final String indexedFieldString, final boolean predefinedField) {
+ final List<SearchableField> searchableFields = new ArrayList<>();
+ if (indexedFieldString != null) {
+ final String[] split = indexedFieldString.split(",");
+ for (String fieldName : split) {
+ fieldName = fieldName.trim();
+ if (fieldName.isEmpty()) {
+ continue;
+ }
+
+ final SearchableField searchableField;
+ if (predefinedField) {
+ searchableField = SearchableFields.getSearchableField(fieldName);
+ } else {
+ searchableField = SearchableFields.newSearchableAttribute(fieldName);
+ }
+
+ if (searchableField == null) {
+ throw new RuntimeException("Invalid Configuration: Provenance Repository configured to Index field '" + fieldName + "', but this is not a valid field");
+ }
+ searchableFields.add(searchableField);
+ }
+ }
+
+ return searchableFields;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFields.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFields.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFields.java
new file mode 100644
index 0000000..de62bca
--- /dev/null
+++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFields.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import org.apache.nifi.provenance.search.SearchableField;
+import org.apache.nifi.provenance.search.SearchableFieldType;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ *
+ */
+public class SearchableFields {
+
+ public static final SearchableField Identifier = new NamedSearchableField("Identifier", "identifier", "Identifier", false);
+ public static final SearchableField EventTime = new NamedSearchableField("EventTime", "time", "Event Time", false, SearchableFieldType.DATE);
+ public static final SearchableField FlowFileUUID = new NamedSearchableField("FlowFileUUID", "uuid", "FlowFile UUID", false);
+ public static final SearchableField Filename = new NamedSearchableField("Filename", "filename", "Filename", false);
+ public static final SearchableField EventType = new NamedSearchableField("EventType", "eventType", "Event Type", false);
+ public static final SearchableField TransitURI = new NamedSearchableField("TransitURI", "transitUri", "Transit URI", false);
+ public static final SearchableField ComponentID = new NamedSearchableField("ProcessorID", "processorId", "Component ID", false);
+ public static final SearchableField AlternateIdentifierURI = new NamedSearchableField("AlternateIdentifierURI", "alternateIdentifierUri", "Alternate Identifier URI", false);
+ public static final SearchableField FileSize = new NamedSearchableField("FileSize", "fileSize", "File Size", false, SearchableFieldType.DATA_SIZE);
+ public static final SearchableField Details = new NamedSearchableField("Details", "details", "Details", false, SearchableFieldType.STRING);
+ public static final SearchableField Relationship = new NamedSearchableField("Relationship", "relationship", "Relationship", false, SearchableFieldType.STRING);
+
+ public static final SearchableField LineageStartDate
+ = new NamedSearchableField("LineageStartDate", "lineageStartDate", "Lineage Start Date", false, SearchableFieldType.DATE);
+ public static final SearchableField LineageIdentifier
+ = new NamedSearchableField("LineageIdentifiers", "lineageIdentifier", "Lineage Identifier", false, SearchableFieldType.STRING);
+
+ public static final SearchableField ContentClaimSection
+ = new NamedSearchableField("ContentClaimSection", "contentClaimSection", "Content Claim Section", false, SearchableFieldType.STRING);
+ public static final SearchableField ContentClaimContainer
+ = new NamedSearchableField("ContentClaimContainer", "contentClaimContainer", "Content Claim Container", false, SearchableFieldType.STRING);
+ public static final SearchableField ContentClaimIdentifier
+ = new NamedSearchableField("ContentClaimIdentifier", "contentClaimIdentifier", "Content Claim Identifier", false, SearchableFieldType.STRING);
+ public static final SearchableField ContentClaimOffset
+ = new NamedSearchableField("ContentClaimOffset", "contentClaimOffset", "Content Claim Offset", false, SearchableFieldType.LONG);
+ public static final SearchableField SourceQueueIdentifier
+ = new NamedSearchableField("SourceQueueIdentifier", "sourceQueueIdentifier", "Source Queue Identifier", false, SearchableFieldType.STRING);
+
+ private static final Map<String, SearchableField> standardFields;
+
+ static {
+ final SearchableField[] searchableFields = new SearchableField[]{
+ EventTime, FlowFileUUID, Filename, EventType, TransitURI,
+ ComponentID, AlternateIdentifierURI, FileSize, Relationship, Details,
+ LineageStartDate, LineageIdentifier, ContentClaimSection, ContentClaimContainer, ContentClaimIdentifier,
+ ContentClaimOffset, SourceQueueIdentifier};
+
+ final Map<String, SearchableField> fields = new HashMap<>();
+ for (final SearchableField field : searchableFields) {
+ fields.put(field.getIdentifier(), field);
+ }
+
+ standardFields = Collections.unmodifiableMap(fields);
+ }
+
+ private SearchableFields() {
+ }
+
+ public static Collection<SearchableField> getStandardFields() {
+ return standardFields.values();
+ }
+
+ public static SearchableField getSearchableField(final String fieldIdentifier) {
+ return standardFields.get(fieldIdentifier);
+ }
+
+ public static SearchableField newSearchableAttribute(final String attributeName) {
+ return new NamedSearchableField(attributeName, attributeName, attributeName, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardLineageResult.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardLineageResult.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardLineageResult.java
new file mode 100644
index 0000000..63c53d0
--- /dev/null
+++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardLineageResult.java
@@ -0,0 +1,326 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.provenance;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.nifi.provenance.lineage.ComputeLineageResult;
+import org.apache.nifi.provenance.lineage.EdgeNode;
+import org.apache.nifi.provenance.lineage.EventNode;
+import org.apache.nifi.provenance.lineage.FlowFileNode;
+import org.apache.nifi.provenance.lineage.LineageEdge;
+import org.apache.nifi.provenance.lineage.LineageNode;
+
+/**
+ *
+ */
+public class StandardLineageResult implements ComputeLineageResult {
+
+ public static final int TTL = (int) TimeUnit.MILLISECONDS.convert(30, TimeUnit.MINUTES);
+ private static final Logger logger = LoggerFactory.getLogger(StandardLineageResult.class);
+
+ private final Collection<String> flowFileUuids;
+ private final Collection<ProvenanceEventRecord> relevantRecords = new ArrayList<>();
+ private final Set<LineageNode> nodes = new HashSet<>();
+ private final Set<LineageEdge> edges = new HashSet<>();
+ private final int numSteps;
+ private final long creationNanos;
+ private long computationNanos;
+
+ private final ReadWriteLock rwLock = new ReentrantReadWriteLock();
+ private final Lock readLock = rwLock.readLock();
+ private final Lock writeLock = rwLock.writeLock();
+
+ private Date expirationDate = null;
+ private String error = null;
+ private int numCompletedSteps = 0;
+
+ private volatile boolean canceled = false;
+
+ public StandardLineageResult(final int numSteps, final Collection<String> flowFileUuids) {
+ this.numSteps = numSteps;
+ this.creationNanos = System.nanoTime();
+ this.flowFileUuids = flowFileUuids;
+
+ updateExpiration();
+ }
+
+ @Override
+ public List<LineageNode> getNodes() {
+ readLock.lock();
+ try {
+ return new ArrayList<>(nodes);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public List<LineageEdge> getEdges() {
+ readLock.lock();
+ try {
+ return new ArrayList<>(edges);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ public int getNumberOfEdges() {
+ readLock.lock();
+ try {
+ return edges.size();
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ public int getNumberOfNodes() {
+ readLock.lock();
+ try {
+ return nodes.size();
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ public long getComputationTime(final TimeUnit timeUnit) {
+ readLock.lock();
+ try {
+ return timeUnit.convert(computationNanos, TimeUnit.NANOSECONDS);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public Date getExpiration() {
+ readLock.lock();
+ try {
+ return expirationDate;
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public String getError() {
+ readLock.lock();
+ try {
+ return error;
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public int getPercentComplete() {
+ readLock.lock();
+ try {
+ return (numSteps < 1) ? 100 : (int) (((float) numCompletedSteps / (float) numSteps) * 100.0F);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public boolean isFinished() {
+ readLock.lock();
+ try {
+ return numCompletedSteps >= numSteps || canceled;
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ public void setError(final String error) {
+ writeLock.lock();
+ try {
+ this.error = error;
+ numCompletedSteps++;
+
+ updateExpiration();
+
+ if (numCompletedSteps >= numSteps) {
+ computationNanos = System.nanoTime() - creationNanos;
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ public void update(final Collection<ProvenanceEventRecord> records) {
+ writeLock.lock();
+ try {
+ relevantRecords.addAll(records);
+
+ numCompletedSteps++;
+ updateExpiration();
+
+ if (numCompletedSteps >= numSteps && error == null) {
+ computeLineage();
+ computationNanos = System.nanoTime() - creationNanos;
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ /**
+ * Computes the lineage from the relevant Provenance Event Records. This
+ * method must be called with the write lock held and is only going to be
+ * useful after all of the records have been successfully obtained
+ */
+ private void computeLineage() {
+ final long startNanos = System.nanoTime();
+
+ nodes.clear();
+ edges.clear();
+
+ Map<String, LineageNode> lastEventMap = new HashMap<>(); // maps FlowFile UUID to last event for that FlowFile
+ final List<ProvenanceEventRecord> sortedRecords = new ArrayList<>(relevantRecords);
+ Collections.sort(sortedRecords, new Comparator<ProvenanceEventRecord>() {
+ @Override
+ public int compare(final ProvenanceEventRecord o1, final ProvenanceEventRecord o2) {
+ // Sort on Event Time, then Event ID.
+ final int eventTimeComparison = Long.compare(o1.getEventTime(), o2.getEventTime());
+ if (eventTimeComparison == 0) {
+ return Long.compare(o1.getEventId(), o2.getEventId());
+ } else {
+ return eventTimeComparison;
+ }
+ }
+ });
+
+ // convert the StandardProvenanceRecord objects into Lineage nodes (FlowFileNode, EventNodes).
+ for (final ProvenanceEventRecord record : sortedRecords) {
+ final LineageNode lineageNode = new EventNode(record);
+ final boolean added = nodes.add(lineageNode);
+ if (!added) {
+ logger.debug("Did not add {} because it already exists in the 'nodes' set", lineageNode);
+ }
+
+ // Create an edge that connects this node to the previous node for the same FlowFile UUID.
+ final LineageNode lastNode = lastEventMap.get(record.getFlowFileUuid());
+ if (lastNode != null) {
+ // We calculate the Edge UUID based on whether or not this event is a SPAWN.
+ // If this event is a SPAWN, then we want to use the previous node's UUID because a
+ // SPAWN Event's UUID is not necessarily what we want, since a SPAWN Event's UUID pertains to
+ // only one of (potentially) many UUIDs associated with the event. Otherwise, we know that
+ // the UUID of this record is appropriate, so we just use it.
+ final String edgeUuid;
+
+ switch (record.getEventType()) {
+ case JOIN:
+ case CLONE:
+ case REPLAY:
+ edgeUuid = lastNode.getFlowFileUuid();
+ break;
+ default:
+ edgeUuid = record.getFlowFileUuid();
+ break;
+ }
+
+ edges.add(new EdgeNode(edgeUuid, lastNode, lineageNode));
+ }
+
+ lastEventMap.put(record.getFlowFileUuid(), lineageNode);
+
+ switch (record.getEventType()) {
+ case FORK:
+ case JOIN:
+ case REPLAY:
+ case CLONE: {
+ // For events that create FlowFile nodes, we need to create the FlowFile Nodes and associated Edges, as appropriate
+ for (final String childUuid : record.getChildUuids()) {
+ if (flowFileUuids.contains(childUuid)) {
+ final FlowFileNode childNode = new FlowFileNode(childUuid, record.getEventTime());
+ final boolean isNewFlowFile = nodes.add(childNode);
+ if (!isNewFlowFile) {
+ final String msg = "Unable to generate Lineage Graph because multiple "
+ + "events were registered claiming to have generated the same FlowFile (UUID = " + childNode.getFlowFileUuid() + ")";
+ logger.error(msg);
+ setError(msg);
+ return;
+ }
+
+ edges.add(new EdgeNode(childNode.getFlowFileUuid(), lineageNode, childNode));
+ lastEventMap.put(childUuid, childNode);
+ }
+ }
+ for (final String parentUuid : record.getParentUuids()) {
+ LineageNode lastNodeForParent = lastEventMap.get(parentUuid);
+ if (lastNodeForParent != null && !lastNodeForParent.equals(lineageNode)) {
+ edges.add(new EdgeNode(parentUuid, lastNodeForParent, lineageNode));
+ }
+
+ lastEventMap.put(parentUuid, lineageNode);
+ }
+ }
+ break;
+ case RECEIVE:
+ case CREATE: {
+ // for a receive event, we want to create a FlowFile Node that represents the FlowFile received
+ // and create an edge from the Receive Event to the FlowFile Node
+ final LineageNode flowFileNode = new FlowFileNode(record.getFlowFileUuid(), record.getEventTime());
+ final boolean isNewFlowFile = nodes.add(flowFileNode);
+ if (!isNewFlowFile) {
+ final String msg = "Found cycle in graph. This indicates that multiple events "
+ + "were registered claiming to have generated the same FlowFile (UUID = " + flowFileNode.getFlowFileUuid() + ")";
+ setError(msg);
+ logger.error(msg);
+ return;
+ }
+ edges.add(new EdgeNode(record.getFlowFileUuid(), lineageNode, flowFileNode));
+ lastEventMap.put(record.getFlowFileUuid(), flowFileNode);
+ }
+ break;
+ default:
+ break;
+ }
+ }
+
+ final long nanos = System.nanoTime() - startNanos;
+ logger.debug("Finished building lineage with {} nodes and {} edges in {} millis", nodes.size(), edges.size(), TimeUnit.NANOSECONDS.toMillis(nanos));
+ }
+
+ void cancel() {
+ this.canceled = true;
+ }
+
+ /**
+ * Must be called with write lock!
+ */
+ private void updateExpiration() {
+ expirationDate = new Date(System.currentTimeMillis() + TTL);
+ }
+}