You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by mc...@apache.org on 2015/01/19 19:16:24 UTC
[50/59] [abbrv] [partial] incubator-nifi git commit: Reworked overall
directory structure to make releasing nifi vs maven plugis easier
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/300952a9/assembly/pom.xml
----------------------------------------------------------------------
diff --git a/assembly/pom.xml b/assembly/pom.xml
deleted file mode 100644
index e785c45..0000000
--- a/assembly/pom.xml
+++ /dev/null
@@ -1,458 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.nifi</groupId>
- <artifactId>nifi-parent</artifactId>
- <version>0.0.1-SNAPSHOT</version>
- </parent>
- <artifactId>nifi</artifactId>
- <version>0.0.1-SNAPSHOT</version>
- <packaging>pom</packaging>
- <name>NiFi Release</name>
- <description>This is the assembly Apache NiFi (incubating)</description>
- <build>
- <plugins>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <configuration>
- <attach>true</attach>
- </configuration>
- <executions>
- <execution>
- <id>make shared resource</id>
- <goals>
- <goal>single</goal>
- </goals>
- <phase>package</phase>
- <configuration>
- <descriptors>
- <descriptor>src/main/assembly/dependencies.xml</descriptor>
- </descriptors>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- <dependencies>
- <dependency>
- <groupId>ch.qos.logback</groupId>
- <artifactId>logback-classic</artifactId>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>jcl-over-slf4j</artifactId>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>jul-to-slf4j</artifactId>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>log4j-over-slf4j</artifactId>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
- <artifactId>nifi-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
- <artifactId>nifi-runtime</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
- <artifactId>nifi-bootstrap</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
- <artifactId>nifi-resources</artifactId>
- <classifier>resources</classifier>
- <scope>runtime</scope>
- <type>zip</type>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
- <artifactId>nifi-docs</artifactId>
- <classifier>resources</classifier>
- <scope>runtime</scope>
- <type>zip</type>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
- <artifactId>nifi-framework-nar</artifactId>
- <type>nar</type>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
- <artifactId>volatile-provenance-repository-nar</artifactId>
- <type>nar</type>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
- <artifactId>persistent-provenance-repository-nar</artifactId>
- <type>nar</type>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
- <artifactId>standard-services-api-nar</artifactId>
- <type>nar</type>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
- <artifactId>ssl-context-service-nar</artifactId>
- <type>nar</type>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
- <artifactId>distributed-cache-services-nar</artifactId>
- <type>nar</type>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
- <artifactId>nifi-standard-nar</artifactId>
- <type>nar</type>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
- <artifactId>nifi-jetty-bundle</artifactId>
- <type>nar</type>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
- <artifactId>update-attribute-nar</artifactId>
- <type>nar</type>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
- <artifactId>monitor-threshold-nar</artifactId>
- <type>nar</type>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
- <artifactId>hadoop-libraries-nar</artifactId>
- <type>nar</type>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
- <artifactId>hadoop-nar</artifactId>
- <type>nar</type>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
- <artifactId>kafka-nar</artifactId>
- <type>nar</type>
- </dependency>
- </dependencies>
-
- <properties>
- <!--Wrapper Properties-->
- <nifi.wrapper.jvm.heap.initial.mb>256</nifi.wrapper.jvm.heap.initial.mb>
- <nifi.wrapper.jvm.heap.max.mb>512</nifi.wrapper.jvm.heap.max.mb>
- <nifi.initial.permgen.size.mb>128</nifi.initial.permgen.size.mb>
- <nifi.max.permgen.size.mb>128</nifi.max.permgen.size.mb>
- <nifi.wrapper.logfile.maxsize>10m</nifi.wrapper.logfile.maxsize>
- <nifi.wrapper.logfile.maxfiles>10</nifi.wrapper.logfile.maxfiles>
-
- <!-- nifi.properties: core properties -->
- <nifi.flowcontroller.autoResumeState>true</nifi.flowcontroller.autoResumeState>
- <nifi.flowcontroller.graceful.shutdown.period>10 sec</nifi.flowcontroller.graceful.shutdown.period>
- <nifi.flowservice.writedelay.interval>500 ms</nifi.flowservice.writedelay.interval>
- <nifi.administrative.yield.duration>30 sec</nifi.administrative.yield.duration>
-
- <nifi.flow.configuration.file>./conf/flow.xml.gz</nifi.flow.configuration.file>
- <nifi.flow.configuration.archive.dir>./conf/archive/</nifi.flow.configuration.archive.dir>
- <nifi.reporting.task.configuration.file>./conf/reporting-tasks.xml</nifi.reporting.task.configuration.file>
- <nifi.controller.service.configuration.file>./conf/controller-services.xml</nifi.controller.service.configuration.file>
- <nifi.authority.provider.configuration.file>./conf/authority-providers.xml</nifi.authority.provider.configuration.file>
- <nifi.templates.directory>./conf/templates</nifi.templates.directory>
- <nifi.database.directory>./database_repository</nifi.database.directory>
-
- <nifi.flowfile.repository.implementation>org.apache.nifi.controller.repository.WriteAheadFlowFileRepository</nifi.flowfile.repository.implementation>
- <nifi.flowfile.repository.directory>./flowfile_repository</nifi.flowfile.repository.directory>
- <nifi.flowfile.repository.partitions>256</nifi.flowfile.repository.partitions>
- <nifi.flowfile.repository.checkpoint.interval>2 mins</nifi.flowfile.repository.checkpoint.interval>
- <nifi.flowfile.repository.always.sync>false</nifi.flowfile.repository.always.sync>
- <nifi.swap.manager.implementation>org.apache.nifi.controller.FileSystemSwapManager</nifi.swap.manager.implementation>
- <nifi.queue.swap.threshold>20000</nifi.queue.swap.threshold>
- <nifi.swap.in.period>5 sec</nifi.swap.in.period>
- <nifi.swap.in.threads>1</nifi.swap.in.threads>
- <nifi.swap.out.period>5 sec</nifi.swap.out.period>
- <nifi.swap.out.threads>4</nifi.swap.out.threads>
-
- <nifi.content.repository.implementation>org.apache.nifi.controller.repository.FileSystemRepository</nifi.content.repository.implementation>
- <nifi.content.claim.max.appendable.size>10 MB</nifi.content.claim.max.appendable.size>
- <nifi.content.claim.max.flow.files>100</nifi.content.claim.max.flow.files>
- <nifi.content.repository.directory.default>./content_repository</nifi.content.repository.directory.default>
- <nifi.content.repository.archive.max.retention.period />
- <nifi.content.repository.archive.max.usage.percentage />
- <nifi.content.repository.archive.enabled>false</nifi.content.repository.archive.enabled>
- <nifi.content.repository.always.sync>false</nifi.content.repository.always.sync>
- <nifi.content.viewer.url />
-
-
- <nifi.restore.directory />
- <nifi.ui.banner.text></nifi.ui.banner.text>
- <nifi.ui.autorefresh.interval>30 sec</nifi.ui.autorefresh.interval>
- <nifi.nar.library.directory>./lib</nifi.nar.library.directory>
- <nifi.nar.working.directory>./work/nar/</nifi.nar.working.directory>
- <nifi.documentation.working.directory>./work/docs/components</nifi.documentation.working.directory>
-
- <nifi.sensitive.props.algorithm>PBEWITHMD5AND256BITAES-CBC-OPENSSL</nifi.sensitive.props.algorithm>
- <nifi.sensitive.props.provider>BC</nifi.sensitive.props.provider>
- <nifi.h2.url.append>;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE</nifi.h2.url.append>
-
- <nifi.remote.input.socket.port>9990</nifi.remote.input.socket.port>
-
- <!-- persistent provenance repository properties -->
- <nifi.provenance.repository.implementation>org.apache.nifi.provenance.PersistentProvenanceRepository</nifi.provenance.repository.implementation>
- <nifi.provenance.repository.directory.default>./provenance_repository</nifi.provenance.repository.directory.default>
- <nifi.provenance.repository.max.storage.time>24 hours</nifi.provenance.repository.max.storage.time>
- <nifi.provenance.repository.max.storage.size>1 GB</nifi.provenance.repository.max.storage.size>
- <nifi.provenance.repository.rollover.time>5 mins</nifi.provenance.repository.rollover.time>
- <nifi.provenance.repository.rollover.size>100 MB</nifi.provenance.repository.rollover.size>
- <nifi.provenance.repository.query.threads>2</nifi.provenance.repository.query.threads>
- <nifi.provenance.repository.compress.on.rollover>true</nifi.provenance.repository.compress.on.rollover>
- <nifi.provenance.repository.indexed.fields>EventType, FlowFileUUID, Filename, ProcessorID</nifi.provenance.repository.indexed.fields>
- <nifi.provenance.repository.indexed.attributes />
- <nifi.provenance.repository.index.shard.size>500 MB</nifi.provenance.repository.index.shard.size>
- <nifi.provenance.repository.always.sync>false</nifi.provenance.repository.always.sync>
- <nifi.provenance.repository.journal.count>16</nifi.provenance.repository.journal.count>
-
- <!-- volatile provenance repository properties -->
- <nifi.provenance.repository.buffer.size>100000</nifi.provenance.repository.buffer.size>
-
- <!-- Component status repository properties -->
- <nifi.components.status.repository.implementation>org.apache.nifi.controller.status.history.VolatileComponentStatusRepository</nifi.components.status.repository.implementation>
- <nifi.components.status.repository.buffer.size>288</nifi.components.status.repository.buffer.size>
- <nifi.components.status.snapshot.frequency>5 mins</nifi.components.status.snapshot.frequency>
-
- <!-- nifi.properties: web properties -->
- <nifi.web.war.directory>./lib</nifi.web.war.directory>
- <nifi.web.http.host />
- <nifi.web.http.port>8080</nifi.web.http.port>
- <nifi.web.https.host />
- <nifi.web.https.port />
- <nifi.jetty.work.dir>./work/jetty</nifi.jetty.work.dir>
- <nifi.web.jetty.threads>200</nifi.web.jetty.threads>
-
- <!-- nifi.properties: security properties -->
- <nifi.security.keystore />
- <nifi.security.keystoreType />
- <nifi.security.keystorePasswd />
- <nifi.security.keyPasswd />
- <nifi.security.truststore />
- <nifi.security.truststoreType />
- <nifi.security.truststorePasswd />
- <nifi.security.needClientAuth />
- <nifi.security.authorizedUsers.file>./conf/authorized-users.xml</nifi.security.authorizedUsers.file>
- <nifi.security.user.credential.cache.duration>24 hours</nifi.security.user.credential.cache.duration>
- <nifi.security.user.authority.provider>file-provider</nifi.security.user.authority.provider>
- <nifi.security.x509.principal.extractor />
- <nifi.security.support.new.account.requests />
- <nifi.security.ocsp.responder.url />
- <nifi.security.ocsp.responder.certificate />
-
- <!-- nifi.properties: cluster common properties (cluster manager and nodes must have same values) -->
- <nifi.cluster.protocol.heartbeat.interval>5 sec</nifi.cluster.protocol.heartbeat.interval>
- <nifi.cluster.protocol.is.secure>false</nifi.cluster.protocol.is.secure>
- <nifi.cluster.protocol.socket.timeout>30 sec</nifi.cluster.protocol.socket.timeout>
- <nifi.cluster.protocol.connection.handshake.timeout>45 sec</nifi.cluster.protocol.connection.handshake.timeout>
- <nifi.cluster.protocol.use.multicast>false</nifi.cluster.protocol.use.multicast>
- <nifi.cluster.protocol.multicast.address />
- <nifi.cluster.protocol.multicast.port />
- <nifi.cluster.protocol.multicast.service.broadcast.delay>500 ms</nifi.cluster.protocol.multicast.service.broadcast.delay>
- <nifi.cluster.protocol.multicast.service.locator.attempts>3</nifi.cluster.protocol.multicast.service.locator.attempts>
- <nifi.cluster.protocol.multicast.service.locator.attempts.delay>1 sec</nifi.cluster.protocol.multicast.service.locator.attempts.delay>
-
- <!-- nifi.properties: cluster node properties (only configure for cluster nodes) -->
- <nifi.cluster.is.node>false</nifi.cluster.is.node>
- <nifi.cluster.node.address />
- <nifi.cluster.node.protocol.port />
- <nifi.cluster.node.protocol.threads>2</nifi.cluster.node.protocol.threads>
- <nifi.cluster.node.unicast.manager.address />
- <nifi.cluster.node.unicast.manager.protocol.port />
-
- <!-- nifi.properties: cluster manager properties (only configure for cluster manager) -->
- <nifi.cluster.is.manager>false</nifi.cluster.is.manager>
- <nifi.cluster.manager.address />
- <nifi.cluster.manager.protocol.port />
- <nifi.cluster.manager.node.firewall.file />
- <nifi.cluster.manager.node.event.history.size>10</nifi.cluster.manager.node.event.history.size>
- <nifi.cluster.manager.node.api.connection.timeout>30 sec</nifi.cluster.manager.node.api.connection.timeout>
- <nifi.cluster.manager.node.api.read.timeout>30 sec</nifi.cluster.manager.node.api.read.timeout>
- <nifi.cluster.manager.node.api.request.threads>10</nifi.cluster.manager.node.api.request.threads>
- <nifi.cluster.manager.flow.retrieval.delay>5 sec</nifi.cluster.manager.flow.retrieval.delay>
- <nifi.cluster.manager.protocol.threads>10</nifi.cluster.manager.protocol.threads>
- <nifi.cluster.manager.safemode.duration>0 sec</nifi.cluster.manager.safemode.duration>
- </properties>
- <profiles>
- <profile>
- <id>rpm</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <build>
- <plugins>
- <plugin>
- <artifactId>maven-dependency-plugin</artifactId>
- <executions>
- <execution>
- <id>unpack-shared-resources</id>
- <goals>
- <goal>unpack-dependencies</goal>
- </goals>
- <phase>generate-resources</phase>
- <configuration>
- <outputDirectory>${project.build.directory}/generated-resources</outputDirectory>
- <includeArtifactIds>nifi-resources</includeArtifactIds>
- <includeGroupIds>org.apache.nifi</includeGroupIds>
- <excludeTransitive>false</excludeTransitive>
- </configuration>
- </execution>
- <execution>
- <id>unpack-docs</id>
- <goals>
- <goal>unpack-dependencies</goal>
- </goals>
- <phase>generate-resources</phase>
- <configuration>
- <outputDirectory>${project.build.directory}/generated-docs</outputDirectory>
- <includeArtifactIds>nifi-docs</includeArtifactIds>
- <includeGroupIds>org.apache.nifi</includeGroupIds>
- <excludeTransitive>false</excludeTransitive>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>rpm-maven-plugin</artifactId>
- <configuration>
- <summary>Apache NiFi (incubating)</summary>
- <description>Apache Nifi (incubating) is dataflow system based on the Flow-Based Programming concepts.</description>
- <license>Apache License, Version 2.0 and others (see included LICENSE file)</license>
- <url>http://nifi.incubator.apache.org</url>
- <group>Utilities</group>
- <prefix>/opt/nifi</prefix>
- <defineStatements>
- <defineStatement>_use_internal_dependency_generator 0</defineStatement>
- </defineStatements>
- <defaultDirmode>750</defaultDirmode>
- <defaultFilemode>640</defaultFilemode>
- <defaultUsername>root</defaultUsername>
- <defaultGroupname>root</defaultGroupname>
- </configuration>
- <executions>
- <execution>
- <id>build-bin-rpm</id>
- <goals>
- <goal>attached-rpm</goal>
- </goals>
- <configuration>
- <classifier>bin</classifier>
- <provides>
- <provide>nifi</provide>
- </provides>
- <mappings>
- <mapping>
- <directory>/opt/nifi/nifi-${project.version}</directory>
- </mapping>
- <mapping>
- <directory>/opt/nifi/nifi-${project.version}</directory>
- <sources>
- <source>
- <location>../LICENSE</location>
- </source>
- <source>
- <location>../NOTICE</location>
- </source>
- <source>
- <location>../DISCLAIMER</location>
- </source>
- <source>
- <location>../README.md</location>
- <destination>README</destination>
- </source>
- </sources>
- </mapping>
- <mapping>
- <directory>/opt/nifi/nifi-${project.version}/bin</directory>
- <filemode>750</filemode>
- <sources>
- <source>
- <location>${project.build.directory}/generated-resources/bin/nifi.sh</location>
- <destination>nifi.sh</destination>
- <filter>true</filter>
- </source>
- </sources>
- </mapping>
- <mapping>
- <directory>/opt/nifi/nifi-${project.version}/conf</directory>
- <configuration>true</configuration>
- <sources>
- <source>
- <location>${project.build.directory}/generated-resources/conf</location>
- <filter>true</filter>
- </source>
- </sources>
- </mapping>
- <mapping>
- <directory>/opt/nifi/nifi-${project.version}/lib</directory>
- <dependency>
- <excludes>
- <exclude>org.apache.nifi:nifi-bootstrap</exclude>
- <exclude>org.apache.nifi:nifi-resources</exclude>
- <exclude>org.apache.nifi:nifi-docs</exclude>
- </excludes>
- </dependency>
- </mapping>
- <mapping>
- <directory>/opt/nifi/nifi-${project.version}/lib/bootstrap</directory>
- <dependency>
- <includes>
- <include>org.apache.nifi:nifi-bootstrap</include>
- </includes>
- </dependency>
- </mapping>
- <mapping>
- <directory>/opt/nifi/nifi-${project.version}/docs</directory>
- <sources>
- <source>
- <location>${project.build.directory}/generated-docs</location>
- </source>
- </sources>
- </mapping>
- </mappings>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
- </profiles>
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/300952a9/assembly/src/main/assembly/dependencies.xml
----------------------------------------------------------------------
diff --git a/assembly/src/main/assembly/dependencies.xml b/assembly/src/main/assembly/dependencies.xml
deleted file mode 100644
index 3481b0a..0000000
--- a/assembly/src/main/assembly/dependencies.xml
+++ /dev/null
@@ -1,140 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<assembly>
- <id>bin</id>
- <formats>
- <format>dir</format>
- <format>zip</format>
- <format>tar.gz</format>
- </formats>
- <includeBaseDirectory>true</includeBaseDirectory>
- <baseDirectory>${project.artifactId}-${project.version}</baseDirectory>
-
- <dependencySets>
- <!-- Write out all dependency artifacts to lib directory -->
- <dependencySet>
- <scope>runtime</scope>
- <useProjectArtifact>false</useProjectArtifact>
- <outputDirectory>lib</outputDirectory>
- <directoryMode>0750</directoryMode>
- <fileMode>0640</fileMode>
- <useTransitiveFiltering>true</useTransitiveFiltering>
- <excludes>
- <exclude>nifi-bootstrap</exclude>
- <exclude>nifi-resources</exclude>
- <exclude>nifi-docs</exclude>
- </excludes>
- </dependencySet>
-
- <!-- Write out the bootstrap lib component to its own dir -->
- <dependencySet>
- <scope>runtime</scope>
- <useProjectArtifact>false</useProjectArtifact>
- <outputDirectory>lib/bootstrap</outputDirectory>
- <directoryMode>0750</directoryMode>
- <fileMode>0640</fileMode>
- <useTransitiveFiltering>true</useTransitiveFiltering>
- <includes>
- <include>nifi-bootstrap</include>
- </includes>
- </dependencySet>
-
- <!-- Write out the conf directory contents -->
- <dependencySet>
- <scope>runtime</scope>
- <useProjectArtifact>false</useProjectArtifact>
- <outputDirectory>./</outputDirectory>
- <directoryMode>0750</directoryMode>
- <fileMode>0640</fileMode>
- <useTransitiveFiltering>true</useTransitiveFiltering>
- <includes>
- <include>nifi-resources</include>
- </includes>
- <unpack>true</unpack>
- <unpackOptions>
- <filtered>true</filtered>
- <includes>
- <include>conf/*</include>
- </includes>
- </unpackOptions>
- </dependencySet>
-
- <!-- Write out the bin directory contents -->
- <dependencySet>
- <scope>runtime</scope>
- <useProjectArtifact>false</useProjectArtifact>
- <outputDirectory>./</outputDirectory>
- <directoryMode>0750</directoryMode>
- <fileMode>0750</fileMode>
- <useTransitiveFiltering>true</useTransitiveFiltering>
- <includes>
- <include>nifi-resources</include>
- </includes>
- <unpack>true</unpack>
- <unpackOptions>
- <filtered>true</filtered>
- <includes>
- <include>bin/*</include>
- </includes>
- </unpackOptions>
- </dependencySet>
-
- <!-- Writes out the docs directory contents -->
- <dependencySet>
- <scope>runtime</scope>
- <useProjectArtifact>false</useProjectArtifact>
- <outputDirectory>docs/</outputDirectory>
- <useTransitiveFiltering>true</useTransitiveFiltering>
- <includes>
- <include>nifi-docs</include>
- </includes>
- <unpack>true</unpack>
- <unpackOptions>
- <filtered>false</filtered>
- </unpackOptions>
- </dependencySet>
- </dependencySets>
- <files>
- <file>
- <source>../README.md</source>
- <outputDirectory>./</outputDirectory>
- <destName>README</destName>
- <fileMode>0644</fileMode>
- <filtered>true</filtered>
- </file>
- <file>
- <source>../DISCLAIMER</source>
- <outputDirectory>./</outputDirectory>
- <destName>DISCLAIMER</destName>
- <fileMode>0644</fileMode>
- <filtered>true</filtered>
- </file>
- <file>
- <source>../LICENSE</source>
- <outputDirectory>./</outputDirectory>
- <destName>LICENSE</destName>
- <fileMode>0644</fileMode>
- <filtered>true</filtered>
- </file>
- <file>
- <source>../NOTICE</source>
- <outputDirectory>./</outputDirectory>
- <destName>NOTICE</destName>
- <fileMode>0644</fileMode>
- <filtered>true</filtered>
- </file>
- </files>
-</assembly>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/300952a9/commons/data-provenance-utils/.gitignore
----------------------------------------------------------------------
diff --git a/commons/data-provenance-utils/.gitignore b/commons/data-provenance-utils/.gitignore
deleted file mode 100755
index 19f2e00..0000000
--- a/commons/data-provenance-utils/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-/target
-/target
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/300952a9/commons/data-provenance-utils/pom.xml
----------------------------------------------------------------------
diff --git a/commons/data-provenance-utils/pom.xml b/commons/data-provenance-utils/pom.xml
deleted file mode 100644
index 0024b70..0000000
--- a/commons/data-provenance-utils/pom.xml
+++ /dev/null
@@ -1,40 +0,0 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.nifi</groupId>
- <artifactId>nifi-commons-parent</artifactId>
- <version>0.0.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>data-provenance-utils</artifactId>
- <version>0.0.1-SNAPSHOT</version>
- <packaging>jar</packaging>
-
- <name>data-provenance-utils</name>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.nifi</groupId>
- <artifactId>nifi-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
- <artifactId>nifi-utils</artifactId>
- </dependency>
- </dependencies>
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/300952a9/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncLineageSubmission.java
----------------------------------------------------------------------
diff --git a/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncLineageSubmission.java b/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncLineageSubmission.java
deleted file mode 100644
index dc24a93..0000000
--- a/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncLineageSubmission.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.provenance;
-
-import java.util.Collection;
-import java.util.Date;
-import java.util.UUID;
-
-import org.apache.nifi.provenance.lineage.ComputeLineageSubmission;
-import org.apache.nifi.provenance.lineage.LineageComputationType;
-
-/**
- *
- */
-public class AsyncLineageSubmission implements ComputeLineageSubmission {
- private final String lineageIdentifier = UUID.randomUUID().toString();
- private final Date submissionTime = new Date();
-
- private final LineageComputationType computationType;
- private final Long eventId;
- private final Collection<String> lineageFlowFileUuids;
-
- private volatile boolean canceled = false;
-
- private final StandardLineageResult result;
-
- public AsyncLineageSubmission(final LineageComputationType computationType, final Long eventId, final Collection<String> lineageFlowFileUuids, final int numSteps) {
- this.computationType = computationType;
- this.eventId = eventId;
- this.lineageFlowFileUuids = lineageFlowFileUuids;
- this.result = new StandardLineageResult(numSteps, lineageFlowFileUuids);
- }
-
- @Override
- public StandardLineageResult getResult() {
- return result;
- }
-
- @Override
- public Date getSubmissionTime() {
- return submissionTime;
- }
-
- @Override
- public String getLineageIdentifier() {
- return lineageIdentifier;
- }
-
- @Override
- public void cancel() {
- this.canceled = true;
- }
-
- @Override
- public boolean isCanceled() {
- return canceled;
- }
-
- @Override
- public LineageComputationType getLineageComputationType() {
- return computationType;
- }
-
- @Override
- public Long getExpandedEventId() {
- return eventId;
- }
-
- @Override
- public Collection<String> getLineageFlowFileUuids() {
- return lineageFlowFileUuids;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/300952a9/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncQuerySubmission.java
----------------------------------------------------------------------
diff --git a/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncQuerySubmission.java b/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncQuerySubmission.java
deleted file mode 100644
index 4244476..0000000
--- a/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncQuerySubmission.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.provenance;
-
-import java.util.Date;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.nifi.provenance.search.Query;
-import org.apache.nifi.provenance.search.QuerySubmission;
-
-/**
- *
- */
-public class AsyncQuerySubmission implements QuerySubmission {
-
- public static final int TTL = (int) TimeUnit.MILLISECONDS.convert(60, TimeUnit.SECONDS);
-
- private final Date submissionTime = new Date();
- private final Query query;
-
- private volatile boolean canceled = false;
- private final StandardQueryResult queryResult;
-
- /**
- * Constructs an AsyncQuerySubmission with the given query and the given
- * number of steps, indicating how many results must be added to this
- * AsyncQuerySubmission before it is considered finished
- *
- * @param query
- * @param numSteps
- */
- public AsyncQuerySubmission(final Query query, final int numSteps) {
- this.query = query;
- queryResult = new StandardQueryResult(query, numSteps);
- }
-
- @Override
- public Date getSubmissionTime() {
- return submissionTime;
- }
-
- @Override
- public String getQueryIdentifier() {
- return query.getIdentifier();
- }
-
- @Override
- public void cancel() {
- this.canceled = true;
- queryResult.cancel();
- }
-
- @Override
- public boolean isCanceled() {
- return canceled;
- }
-
- @Override
- public Query getQuery() {
- return query;
- }
-
- @Override
- public StandardQueryResult getResult() {
- return queryResult;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/300952a9/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/NamedSearchableField.java
----------------------------------------------------------------------
diff --git a/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/NamedSearchableField.java b/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/NamedSearchableField.java
deleted file mode 100644
index dc2903f..0000000
--- a/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/NamedSearchableField.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.provenance;
-
-import org.apache.nifi.provenance.search.SearchableField;
-import org.apache.nifi.provenance.search.SearchableFieldType;
-
-import static java.util.Objects.requireNonNull;
-
-/**
- *
- */
-public class NamedSearchableField implements SearchableField {
-
- private final String identifier;
- private final String searchableName;
- private final SearchableFieldType fieldType;
- private final String friendlyName;
- private final boolean attribute;
-
- NamedSearchableField(final String identifier, final String searchableName, final String friendlyName, final boolean attribute) {
- this(identifier, searchableName, friendlyName, attribute, SearchableFieldType.STRING);
- }
-
- NamedSearchableField(final String identifier, final String searchableName, final String friendlyName, final boolean attribute, final SearchableFieldType fieldType) {
- this.identifier = requireNonNull(identifier);
- this.searchableName = requireNonNull(searchableName);
- this.friendlyName = requireNonNull(friendlyName);
- this.attribute = requireNonNull(attribute);
- this.fieldType = requireNonNull(fieldType);
- }
-
- @Override
- public String getIdentifier() {
- return identifier;
- }
-
- @Override
- public String getSearchableFieldName() {
- return searchableName;
- }
-
- @Override
- public String getFriendlyName() {
- return friendlyName;
- }
-
- @Override
- public boolean isAttribute() {
- return attribute;
- }
-
- @Override
- public SearchableFieldType getFieldType() {
- return fieldType;
- }
-
- @Override
- public String toString() {
- return friendlyName;
- }
-
- @Override
- public int hashCode() {
- return 298347 + searchableName.hashCode() + (attribute ? 1 : 0);
- }
-
- @Override
- public boolean equals(final Object obj) {
- if (obj == null) {
- return false;
- }
-
- if (!(obj instanceof SearchableField)) {
- return false;
- }
-
- final SearchableField other = (SearchableField) obj;
- return (this.searchableName.equals(other.getSearchableFieldName()) && attribute == other.isAttribute());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/300952a9/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFieldParser.java
----------------------------------------------------------------------
diff --git a/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFieldParser.java b/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFieldParser.java
deleted file mode 100644
index 6a934b1..0000000
--- a/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFieldParser.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.provenance;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.nifi.provenance.search.SearchableField;
-
-public class SearchableFieldParser {
-
- public static List<SearchableField> extractSearchableFields(final String indexedFieldString, final boolean predefinedField) {
- final List<SearchableField> searchableFields = new ArrayList<>();
- if (indexedFieldString != null) {
- final String[] split = indexedFieldString.split(",");
- for (String fieldName : split) {
- fieldName = fieldName.trim();
- if (fieldName.isEmpty()) {
- continue;
- }
-
- final SearchableField searchableField;
- if (predefinedField) {
- searchableField = SearchableFields.getSearchableField(fieldName);
- } else {
- searchableField = SearchableFields.newSearchableAttribute(fieldName);
- }
-
- if (searchableField == null) {
- throw new RuntimeException("Invalid Configuration: Provenance Repository configured to Index field '" + fieldName + "', but this is not a valid field");
- }
- searchableFields.add(searchableField);
- }
- }
-
- return searchableFields;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/300952a9/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFields.java
----------------------------------------------------------------------
diff --git a/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFields.java b/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFields.java
deleted file mode 100644
index 97c9880..0000000
--- a/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFields.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.provenance;
-
-import org.apache.nifi.provenance.search.SearchableField;
-import org.apache.nifi.provenance.search.SearchableFieldType;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- *
- */
-public class SearchableFields {
-
- public static final SearchableField Identifier = new NamedSearchableField("Identifier", "identifier", "Identifier", false);
- public static final SearchableField EventTime = new NamedSearchableField("EventTime", "time", "Event Time", false, SearchableFieldType.DATE);
- public static final SearchableField FlowFileUUID = new NamedSearchableField("FlowFileUUID", "uuid", "FlowFile UUID", false);
- public static final SearchableField Filename = new NamedSearchableField("Filename", "filename", "Filename", false);
- public static final SearchableField EventType = new NamedSearchableField("EventType", "eventType", "Event Type", false);
- public static final SearchableField TransitURI = new NamedSearchableField("TransitURI", "transitUri", "Transit URI", false);
- public static final SearchableField ComponentID = new NamedSearchableField("ProcessorID", "processorId", "Component ID", false);
- public static final SearchableField AlternateIdentifierURI = new NamedSearchableField("AlternateIdentifierURI", "alternateIdentifierUri", "Alternate Identifier URI", false);
- public static final SearchableField FileSize = new NamedSearchableField("FileSize", "fileSize", "File Size", false, SearchableFieldType.DATA_SIZE);
- public static final SearchableField Details = new NamedSearchableField("Details", "details", "Details", false, SearchableFieldType.STRING);
- public static final SearchableField Relationship = new NamedSearchableField("Relationship", "relationship", "Relationship", false, SearchableFieldType.STRING);
-
- public static final SearchableField LineageStartDate = new NamedSearchableField("LineageStartDate", "lineageStartDate", "Lineage Start Date", false, SearchableFieldType.DATE);
- public static final SearchableField LineageIdentifier = new NamedSearchableField("LineageIdentifiers", "lineageIdentifier", "Lineage Identifier", false, SearchableFieldType.STRING);
-
- public static final SearchableField ContentClaimSection = new NamedSearchableField("ContentClaimSection", "contentClaimSection", "Content Claim Section", false, SearchableFieldType.STRING);
- public static final SearchableField ContentClaimContainer = new NamedSearchableField("ContentClaimContainer", "contentClaimContainer", "Content Claim Container", false, SearchableFieldType.STRING);
- public static final SearchableField ContentClaimIdentifier = new NamedSearchableField("ContentClaimIdentifier", "contentClaimIdentifier", "Content Claim Identifier", false, SearchableFieldType.STRING);
- public static final SearchableField ContentClaimOffset = new NamedSearchableField("ContentClaimOffset", "contentClaimOffset", "Content Claim Offset", false, SearchableFieldType.LONG);
- public static final SearchableField SourceQueueIdentifier = new NamedSearchableField("SourceQueueIdentifier", "sourceQueueIdentifier", "Source Queue Identifier", false, SearchableFieldType.STRING);
-
- private static final Map<String, SearchableField> standardFields;
-
- static {
- final SearchableField[] searchableFields = new SearchableField[]{
- EventTime, FlowFileUUID, Filename, EventType, TransitURI,
- ComponentID, AlternateIdentifierURI, FileSize, Relationship, Details,
- LineageStartDate, LineageIdentifier, ContentClaimSection, ContentClaimContainer, ContentClaimIdentifier,
- ContentClaimOffset, SourceQueueIdentifier};
-
- final Map<String, SearchableField> fields = new HashMap<>();
- for (final SearchableField field : searchableFields) {
- fields.put(field.getIdentifier(), field);
- }
-
- standardFields = Collections.unmodifiableMap(fields);
- }
-
- private SearchableFields() {
- }
-
- public static Collection<SearchableField> getStandardFields() {
- return standardFields.values();
- }
-
- public static SearchableField getSearchableField(final String fieldIdentifier) {
- return standardFields.get(fieldIdentifier);
- }
-
- public static SearchableField newSearchableAttribute(final String attributeName) {
- return new NamedSearchableField(attributeName, attributeName, attributeName, true);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/300952a9/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardLineageResult.java
----------------------------------------------------------------------
diff --git a/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardLineageResult.java b/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardLineageResult.java
deleted file mode 100644
index afb56e8..0000000
--- a/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardLineageResult.java
+++ /dev/null
@@ -1,324 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.provenance;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.nifi.provenance.lineage.ComputeLineageResult;
-import org.apache.nifi.provenance.lineage.EdgeNode;
-import org.apache.nifi.provenance.lineage.EventNode;
-import org.apache.nifi.provenance.lineage.FlowFileNode;
-import org.apache.nifi.provenance.lineage.LineageEdge;
-import org.apache.nifi.provenance.lineage.LineageNode;
-
-/**
- *
- */
-public class StandardLineageResult implements ComputeLineageResult {
-
- public static final int TTL = (int) TimeUnit.MILLISECONDS.convert(30, TimeUnit.MINUTES);
- private static final Logger logger = LoggerFactory.getLogger(StandardLineageResult.class);
-
- private final Collection<String> flowFileUuids;
- private final Collection<ProvenanceEventRecord> relevantRecords = new ArrayList<>();
- private final Set<LineageNode> nodes = new HashSet<>();
- private final Set<LineageEdge> edges = new HashSet<>();
- private final int numSteps;
- private final long creationNanos;
- private long computationNanos;
-
- private final ReadWriteLock rwLock = new ReentrantReadWriteLock();
- private final Lock readLock = rwLock.readLock();
- private final Lock writeLock = rwLock.writeLock();
-
- private Date expirationDate = null;
- private String error = null;
- private int numCompletedSteps = 0;
-
- private volatile boolean canceled = false;
-
- public StandardLineageResult(final int numSteps, final Collection<String> flowFileUuids) {
- this.numSteps = numSteps;
- this.creationNanos = System.nanoTime();
- this.flowFileUuids = flowFileUuids;
-
- updateExpiration();
- }
-
- @Override
- public List<LineageNode> getNodes() {
- readLock.lock();
- try {
- return new ArrayList<>(nodes);
- } finally {
- readLock.unlock();
- }
- }
-
- @Override
- public List<LineageEdge> getEdges() {
- readLock.lock();
- try {
- return new ArrayList<>(edges);
- } finally {
- readLock.unlock();
- }
- }
-
- public int getNumberOfEdges() {
- readLock.lock();
- try {
- return edges.size();
- } finally {
- readLock.unlock();
- }
- }
-
- public int getNumberOfNodes() {
- readLock.lock();
- try {
- return nodes.size();
- } finally {
- readLock.unlock();
- }
- }
-
- public long getComputationTime(final TimeUnit timeUnit) {
- readLock.lock();
- try {
- return timeUnit.convert(computationNanos, TimeUnit.NANOSECONDS);
- } finally {
- readLock.unlock();
- }
- }
-
- @Override
- public Date getExpiration() {
- readLock.lock();
- try {
- return expirationDate;
- } finally {
- readLock.unlock();
- }
- }
-
- @Override
- public String getError() {
- readLock.lock();
- try {
- return error;
- } finally {
- readLock.unlock();
- }
- }
-
- @Override
- public int getPercentComplete() {
- readLock.lock();
- try {
- return (numSteps < 1) ? 100 : (int) (((float) numCompletedSteps / (float) numSteps) * 100.0F);
- } finally {
- readLock.unlock();
- }
- }
-
- @Override
- public boolean isFinished() {
- readLock.lock();
- try {
- return numCompletedSteps >= numSteps || canceled;
- } finally {
- readLock.unlock();
- }
- }
-
- public void setError(final String error) {
- writeLock.lock();
- try {
- this.error = error;
- numCompletedSteps++;
-
- updateExpiration();
-
- if (numCompletedSteps >= numSteps) {
- computationNanos = System.nanoTime() - creationNanos;
- }
- } finally {
- writeLock.unlock();
- }
- }
-
- public void update(final Collection<ProvenanceEventRecord> records) {
- writeLock.lock();
- try {
- relevantRecords.addAll(records);
-
- numCompletedSteps++;
- updateExpiration();
-
- if (numCompletedSteps >= numSteps && error == null) {
- computeLineage();
- computationNanos = System.nanoTime() - creationNanos;
- }
- } finally {
- writeLock.unlock();
- }
- }
-
- /**
- * Computes the lineage from the relevant Provenance Event Records. This
- * method must be called with the write lock held and is only going to be
- * useful after all of the records have been successfully obtained
- */
- private void computeLineage() {
- final long startNanos = System.nanoTime();
-
- nodes.clear();
- edges.clear();
-
- Map<String, LineageNode> lastEventMap = new HashMap<>(); // maps FlowFile UUID to last event for that FlowFile
- final List<ProvenanceEventRecord> sortedRecords = new ArrayList<>(relevantRecords);
- Collections.sort(sortedRecords, new Comparator<ProvenanceEventRecord>() {
- @Override
- public int compare(final ProvenanceEventRecord o1, final ProvenanceEventRecord o2) {
- // Sort on Event Time, then Event ID.
- final int eventTimeComparison = Long.compare(o1.getEventTime(), o2.getEventTime());
- if (eventTimeComparison == 0) {
- return Long.compare(o1.getEventId(), o2.getEventId());
- } else {
- return eventTimeComparison;
- }
- }
- });
-
- // convert the StandardProvenanceRecord objects into Lineage nodes (FlowFileNode, EventNodes).
- for (final ProvenanceEventRecord record : sortedRecords) {
- final LineageNode lineageNode = new EventNode(record);
- final boolean added = nodes.add(lineageNode);
- if (!added) {
- logger.debug("Did not add {} because it already exists in the 'nodes' set", lineageNode);
- }
-
- // Create an edge that connects this node to the previous node for the same FlowFile UUID.
- final LineageNode lastNode = lastEventMap.get(record.getFlowFileUuid());
- if (lastNode != null) {
- // We calculate the Edge UUID based on whether or not this event is a SPAWN.
- // If this event is a SPAWN, then we want to use the previous node's UUID because a
- // SPAWN Event's UUID is not necessarily what we want, since a SPAWN Event's UUID pertains to
- // only one of (potentially) many UUIDs associated with the event. Otherwise, we know that
- // the UUID of this record is appropriate, so we just use it.
- final String edgeUuid;
-
- switch (record.getEventType()) {
- case JOIN:
- case CLONE:
- case REPLAY:
- edgeUuid = lastNode.getFlowFileUuid();
- break;
- default:
- edgeUuid = record.getFlowFileUuid();
- break;
- }
-
- edges.add(new EdgeNode(edgeUuid, lastNode, lineageNode));
- }
-
- lastEventMap.put(record.getFlowFileUuid(), lineageNode);
-
- switch (record.getEventType()) {
- case FORK:
- case JOIN:
- case REPLAY:
- case CLONE: {
- // For events that create FlowFile nodes, we need to create the FlowFile Nodes and associated Edges, as appropriate
- for (final String childUuid : record.getChildUuids()) {
- if (flowFileUuids.contains(childUuid)) {
- final FlowFileNode childNode = new FlowFileNode(childUuid, record.getEventTime());
- final boolean isNewFlowFile = nodes.add(childNode);
- if (!isNewFlowFile) {
- final String msg = "Unable to generate Lineage Graph because multiple events were registered claiming to have generated the same FlowFile (UUID = " + childNode.getFlowFileUuid() + ")";
- logger.error(msg);
- setError(msg);
- return;
- }
-
- edges.add(new EdgeNode(childNode.getFlowFileUuid(), lineageNode, childNode));
- lastEventMap.put(childUuid, childNode);
- }
- }
- for (final String parentUuid : record.getParentUuids()) {
- LineageNode lastNodeForParent = lastEventMap.get(parentUuid);
- if (lastNodeForParent != null && !lastNodeForParent.equals(lineageNode)) {
- edges.add(new EdgeNode(parentUuid, lastNodeForParent, lineageNode));
- }
-
- lastEventMap.put(parentUuid, lineageNode);
- }
- }
- break;
- case RECEIVE:
- case CREATE: {
- // for a receive event, we want to create a FlowFile Node that represents the FlowFile received
- // and create an edge from the Receive Event to the FlowFile Node
- final LineageNode flowFileNode = new FlowFileNode(record.getFlowFileUuid(), record.getEventTime());
- final boolean isNewFlowFile = nodes.add(flowFileNode);
- if (!isNewFlowFile) {
- final String msg = "Found cycle in graph. This indicates that multiple events were registered claiming to have generated the same FlowFile (UUID = " + flowFileNode.getFlowFileUuid() + ")";
- setError(msg);
- logger.error(msg);
- return;
- }
- edges.add(new EdgeNode(record.getFlowFileUuid(), lineageNode, flowFileNode));
- lastEventMap.put(record.getFlowFileUuid(), flowFileNode);
- }
- break;
- default:
- break;
- }
- }
-
- final long nanos = System.nanoTime() - startNanos;
- logger.debug("Finished building lineage with {} nodes and {} edges in {} millis", nodes.size(), edges.size(), TimeUnit.NANOSECONDS.toMillis(nanos));
- }
-
- void cancel() {
- this.canceled = true;
- }
-
- /**
- * Must be called with write lock!
- */
- private void updateExpiration() {
- expirationDate = new Date(System.currentTimeMillis() + TTL);
- }
-}