You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by mc...@apache.org on 2015/11/23 21:46:12 UTC
[01/50] [abbrv] nifi git commit: NIFI-1097: Rewrite PutKafka to use
the new producer api
Repository: nifi
Updated Branches:
refs/heads/NIFI-655 5ef53b6fe -> 769f19ee8
NIFI-1097: Rewrite PutKafka to use the new producer api
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/22de23ba
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/22de23ba
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/22de23ba
Branch: refs/heads/NIFI-655
Commit: 22de23baa6ae7318574ccb112d03daf8f8dfa10e
Parents: 7a165b6
Author: Mark Payne <ma...@hotmail.com>
Authored: Tue Nov 3 16:17:32 2015 -0500
Committer: Mark Payne <ma...@hotmail.com>
Committed: Fri Nov 13 10:46:46 2015 -0500
----------------------------------------------------------------------
.../nifi-kafka-processors/pom.xml | 5 +
.../apache/nifi/processors/kafka/PutKafka.java | 712 +++++++++++++------
.../nifi/processors/kafka/TestPutKafka.java | 439 ++++++------
3 files changed, 727 insertions(+), 429 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/22de23ba/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-processors/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-processors/pom.xml b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-processors/pom.xml
index c2e86fe..1a8dc9d 100644
--- a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-processors/pom.xml
+++ b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-processors/pom.xml
@@ -35,6 +35,11 @@
<artifactId>nifi-utils</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ <version>0.8.2.2</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.9.1</artifactId>
<version>0.8.2.2</version>
http://git-wip-us.apache.org/repos/asf/nifi/blob/22de23ba/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/processors/kafka/PutKafka.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/processors/kafka/PutKafka.java b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/processors/kafka/PutKafka.java
index 09025a4..ea7f7bb 100644
--- a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/processors/kafka/PutKafka.java
+++ b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/processors/kafka/PutKafka.java
@@ -21,30 +21,47 @@ import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.regex.Pattern;
+import org.apache.kafka.clients.producer.BufferExhaustedException;
+import org.apache.kafka.clients.producer.Callback;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.Producer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.Node;
+import org.apache.kafka.common.PartitionInfo;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
import org.apache.nifi.annotation.behavior.InputRequirement;
import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.TriggerWhenEmpty;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
import org.apache.nifi.annotation.lifecycle.OnStopped;
import org.apache.nifi.components.AllowableValue;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.ValidationContext;
import org.apache.nifi.components.ValidationResult;
import org.apache.nifi.flowfile.FlowFile;
-import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.AbstractSessionFactoryProcessor;
import org.apache.nifi.processor.DataUnit;
import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.ProcessSessionFactory;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.io.InputStreamCallback;
@@ -56,21 +73,20 @@ import org.apache.nifi.stream.io.StreamUtils;
import org.apache.nifi.stream.io.util.NonThreadSafeCircularBuffer;
import org.apache.nifi.util.LongHolder;
-import kafka.javaapi.producer.Producer;
-import kafka.producer.KeyedMessage;
-import kafka.producer.ProducerConfig;
import scala.actors.threadpool.Arrays;
@SupportsBatching
@InputRequirement(Requirement.INPUT_REQUIRED)
-@Tags({ "Apache", "Kafka", "Put", "Send", "Message", "PubSub" })
-@CapabilityDescription("Sends the contents of a FlowFile as a message to Apache Kafka")
-public class PutKafka extends AbstractProcessor {
+@Tags({"Apache", "Kafka", "Put", "Send", "Message", "PubSub"})
+@CapabilityDescription("Sends the contents of a FlowFile as a message to Apache Kafka. The messages to send may be individual FlowFiles or may be delimited, using a "
+ + "user-specified delimiter, such as a new-line.")
+@TriggerWhenEmpty // because we have a queue of sessions that are ready to be committed
+public class PutKafka extends AbstractSessionFactoryProcessor {
private static final String SINGLE_BROKER_REGEX = ".*?\\:\\d{3,5}";
private static final String BROKER_REGEX = SINGLE_BROKER_REGEX + "(?:,\\s*" + SINGLE_BROKER_REGEX + ")*";
- public static final AllowableValue DELIVERY_REPLICATED = new AllowableValue("-1", "Guarantee Replicated Delivery", "FlowFile will be routed to"
+ public static final AllowableValue DELIVERY_REPLICATED = new AllowableValue("all", "Guarantee Replicated Delivery", "FlowFile will be routed to"
+ " failure unless the message is replicated to the appropriate number of Kafka Nodes according to the Topic configuration");
public static final AllowableValue DELIVERY_ONE_NODE = new AllowableValue("1", "Guarantee Single Node Delivery", "FlowFile will be routed"
+ " to success if the message is received by a single Kafka node, whether or not it is replicated. This is faster than"
@@ -79,16 +95,6 @@ public class PutKafka extends AbstractProcessor {
+ " successfully writing the content to a Kafka node, without waiting for a response. This provides the best performance but may result"
+ " in data loss.");
- /**
- * AllowableValue for a Producer Type that synchronously sends messages to Kafka
- */
- public static final AllowableValue PRODUCTER_TYPE_SYNCHRONOUS = new AllowableValue("sync", "Synchronous", "Send FlowFiles to Kafka immediately.");
-
- /**
- * AllowableValue for a Producer Type that asynchronously sends messages to Kafka
- */
- public static final AllowableValue PRODUCTER_TYPE_ASYNCHRONOUS = new AllowableValue("async", "Asynchronous", "Batch messages before sending them to Kafka."
- + " While this will improve throughput, it opens the possibility that a failure on the client machine will drop unsent data.");
/**
* AllowableValue for sending messages to Kafka without compression
@@ -105,6 +111,13 @@ public class PutKafka extends AbstractProcessor {
*/
public static final AllowableValue COMPRESSION_CODEC_SNAPPY = new AllowableValue("snappy", "Snappy", "Compress messages using Snappy");
+ static final AllowableValue ROUND_ROBIN_PARTITIONING = new AllowableValue("Round Robin", "Round Robin",
+ "Messages will be assigned partitions in a round-robin fashion, sending the first message to Partition 1, the next Partition to Partition 2, and so on, wrapping as necessary.");
+ static final AllowableValue RANDOM_PARTITIONING = new AllowableValue("Random Robin", "Random",
+ "Messages will be assigned to random partitions.");
+ static final AllowableValue USER_DEFINED_PARTITIONING = new AllowableValue("User-Defined", "User-Defined",
+ "The <Partition> property will be used to determine the partition. All messages within the same FlowFile will be assigned to the same partition.");
+
public static final PropertyDescriptor SEED_BROKERS = new PropertyDescriptor.Builder()
.name("Known Brokers")
@@ -120,6 +133,21 @@ public class PutKafka extends AbstractProcessor {
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.expressionLanguageSupported(true)
.build();
+ static final PropertyDescriptor PARTITION_STRATEGY = new PropertyDescriptor.Builder()
+ .name("Partition Strategy")
+ .description("Specifies how messages should be partitioned when sent to Kafka")
+ .allowableValues(ROUND_ROBIN_PARTITIONING, RANDOM_PARTITIONING, USER_DEFINED_PARTITIONING)
+ .defaultValue(ROUND_ROBIN_PARTITIONING.getValue())
+ .required(true)
+ .build();
+ public static final PropertyDescriptor PARTITION = new PropertyDescriptor.Builder()
+ .name("Partition")
+ .description("Specifies which Kafka Partition to add the message to. If using a message delimiter, all messages in the same FlowFile will be sent to the same partition. "
+ + "If a partition is specified but is not valid, then all messages within the same FlowFile will use the same partition but it remains undefined which partition is used.")
+ .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR)
+ .expressionLanguageSupported(true)
+ .required(false)
+ .build();
public static final PropertyDescriptor KEY = new PropertyDescriptor.Builder()
.name("Kafka Key")
.description("The Key to use for the Message")
@@ -140,7 +168,10 @@ public class PutKafka extends AbstractProcessor {
.description("Specifies the delimiter to use for splitting apart multiple messages within a single FlowFile. "
+ "If not specified, the entire content of the FlowFile will be used as a single message. "
+ "If specified, the contents of the FlowFile will be split on this delimiter and each section "
- + "sent as a separate Kafka message.")
+ + "sent as a separate Kafka message. Note that if messages are delimited and some messages for a given FlowFile "
+ + "are transferred successfully while others are not, the messages will be split into individual FlowFiles, such that those "
+ + "messages that were successfully sent are routed to the 'success' relationship while other messages are sent to the 'failure' "
+ + "relationship.")
.required(false)
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.expressionLanguageSupported(true)
@@ -151,6 +182,13 @@ public class PutKafka extends AbstractProcessor {
.required(true)
.addValidator(StandardValidators.DATA_SIZE_VALIDATOR)
.expressionLanguageSupported(false)
+ .defaultValue("5 MB")
+ .build();
+ static final PropertyDescriptor MAX_RECORD_SIZE = new PropertyDescriptor.Builder()
+ .name("Max Record Size")
+ .description("The maximum size that any individual record can be.")
+ .addValidator(StandardValidators.DATA_SIZE_VALIDATOR)
+ .required(true)
.defaultValue("1 MB")
.build();
public static final PropertyDescriptor TIMEOUT = new PropertyDescriptor.Builder()
@@ -168,20 +206,10 @@ public class PutKafka extends AbstractProcessor {
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.expressionLanguageSupported(false)
.build();
- public static final PropertyDescriptor PRODUCER_TYPE = new PropertyDescriptor.Builder()
- .name("Producer Type")
- .description("This parameter specifies whether the messages are sent asynchronously in a background thread.")
- .required(true)
- .allowableValues(PRODUCTER_TYPE_SYNCHRONOUS, PRODUCTER_TYPE_ASYNCHRONOUS)
- .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
- .expressionLanguageSupported(false)
- .defaultValue(PRODUCTER_TYPE_SYNCHRONOUS.getValue())
- .build();
public static final PropertyDescriptor BATCH_NUM_MESSAGES = new PropertyDescriptor.Builder()
.name("Async Batch Size")
- .description("Used only if Producer Type is set to \"" + PRODUCTER_TYPE_ASYNCHRONOUS.getDisplayName() + "\"."
- + " The number of messages to send in one batch when using " + PRODUCTER_TYPE_ASYNCHRONOUS.getDisplayName() + " mode."
- + " The producer will wait until either this number of messages are ready"
+ .displayName("Batch Size")
+ .description("The number of messages to send in one batch. The producer will wait until either this number of messages are ready"
+ " to send or \"Queue Buffering Max Time\" is reached.")
.required(true)
.addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR)
@@ -189,35 +217,13 @@ public class PutKafka extends AbstractProcessor {
.build();
public static final PropertyDescriptor QUEUE_BUFFERING_MAX = new PropertyDescriptor.Builder()
.name("Queue Buffering Max Time")
- .description("Used only if Producer Type is set to \"" + PRODUCTER_TYPE_ASYNCHRONOUS.getDisplayName() + "\"."
- + " Maximum time to buffer data when using " + PRODUCTER_TYPE_ASYNCHRONOUS.getDisplayName() + " mode. For example a setting of 100 ms"
- + " will try to batch together 100ms of messages to send at once. This will improve"
+ .description("Maximum time to buffer data before sending to Kafka. For example a setting of 100 ms"
+ + " will try to batch together 100 milliseconds' worth of messages to send at once. This will improve"
+ " throughput but adds message delivery latency due to the buffering.")
.required(true)
.addValidator(StandardValidators.TIME_PERIOD_VALIDATOR)
.defaultValue("5 secs")
.build();
- public static final PropertyDescriptor QUEUE_BUFFERING_MAX_MESSAGES = new PropertyDescriptor.Builder()
- .name("Queue Buffer Max Count")
- .description("Used only if Producer Type is set to \"" + PRODUCTER_TYPE_ASYNCHRONOUS.getDisplayName() + "\"."
- + " The maximum number of unsent messages that can be queued up in the producer when"
- + " using " + PRODUCTER_TYPE_ASYNCHRONOUS.getDisplayName() + " mode before either the producer must be blocked or data must be dropped.")
- .required(true)
- .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR)
- .defaultValue("10000")
- .build();
- public static final PropertyDescriptor QUEUE_ENQUEUE_TIMEOUT = new PropertyDescriptor.Builder()
- .name("Queue Enqueue Timeout")
- .description("Used only if Producer Type is set to \"" + PRODUCTER_TYPE_ASYNCHRONOUS.getDisplayName() + "\"."
- + " The amount of time to block before dropping messages when running in "
- + PRODUCTER_TYPE_ASYNCHRONOUS.getDisplayName() + " mode"
- + " and the buffer has reached the \"Queue Buffer Max Count\". If set to 0, events will"
- + " be enqueued immediately or dropped if the queue is full (the producer send call will"
- + " never block). If not set, the producer will block indefinitely and never willingly"
- + " drop a send.")
- .required(false)
- .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR)
- .build();
public static final PropertyDescriptor COMPRESSION_CODEC = new PropertyDescriptor.Builder()
.name("Compression Codec")
.description("This parameter allows you to specify the compression codec for all"
@@ -227,16 +233,6 @@ public class PutKafka extends AbstractProcessor {
.allowableValues(COMPRESSION_CODEC_NONE, COMPRESSION_CODEC_GZIP, COMPRESSION_CODEC_SNAPPY)
.defaultValue(COMPRESSION_CODEC_NONE.getValue())
.build();
- public static final PropertyDescriptor COMPRESSED_TOPICS = new PropertyDescriptor.Builder()
- .name("Compressed Topics")
- .description("This parameter allows you to set whether compression should be turned on"
- + " for particular topics. If the compression codec is anything other than"
- + " \"" + COMPRESSION_CODEC_NONE.getDisplayName() + "\", enable compression only for specified topics if any."
- + " If the list of compressed topics is empty, then enable the specified"
- + " compression codec for all topics. If the compression codec is " + COMPRESSION_CODEC_NONE.getDisplayName() + ","
- + " compression is disabled for all topics")
- .required(false)
- .build();
public static final Relationship REL_SUCCESS = new Relationship.Builder()
.name("success")
@@ -247,7 +243,13 @@ public class PutKafka extends AbstractProcessor {
.description("Any FlowFile that cannot be sent to Kafka will be routed to this Relationship")
.build();
- private final BlockingQueue<Producer<byte[], byte[]>> producers = new LinkedBlockingQueue<>();
+ private static final Pattern NUMBER_PATTERN = Pattern.compile("-?\\d+");
+ private final BlockingQueue<FlowFileMessageBatch> completeBatches = new LinkedBlockingQueue<>();
+ private final Set<FlowFileMessageBatch> activeBatches = Collections.synchronizedSet(new HashSet<FlowFileMessageBatch>());
+
+ private final ConcurrentMap<String, AtomicLong> partitionIndexMap = new ConcurrentHashMap<>();
+
+ private volatile Producer<byte[], byte[]> producer;
@Override
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
@@ -259,36 +261,21 @@ public class PutKafka extends AbstractProcessor {
final List<PropertyDescriptor> props = new ArrayList<>();
props.add(SEED_BROKERS);
props.add(TOPIC);
+ props.add(PARTITION_STRATEGY);
+ props.add(PARTITION);
props.add(KEY);
props.add(DELIVERY_GUARANTEE);
props.add(MESSAGE_DELIMITER);
props.add(MAX_BUFFER_SIZE);
+ props.add(MAX_RECORD_SIZE);
props.add(TIMEOUT);
- props.add(PRODUCER_TYPE);
props.add(BATCH_NUM_MESSAGES);
- props.add(QUEUE_BUFFERING_MAX_MESSAGES);
props.add(QUEUE_BUFFERING_MAX);
- props.add(QUEUE_ENQUEUE_TIMEOUT);
props.add(COMPRESSION_CODEC);
- props.add(COMPRESSED_TOPICS);
props.add(clientName);
return props;
}
- @Override
- public Collection<ValidationResult> customValidate(final ValidationContext context) {
- final List<ValidationResult> errors = new ArrayList<>(super.customValidate(context));
-
- final Integer batchMessages = context.getProperty(BATCH_NUM_MESSAGES).asInteger();
- final Integer bufferMaxMessages = context.getProperty(QUEUE_BUFFERING_MAX_MESSAGES).asInteger();
-
- if (batchMessages > bufferMaxMessages) {
- errors.add(new ValidationResult.Builder().subject("Batch Size, Queue Buffer").valid(false)
- .explanation("Batch Size (" + batchMessages + ") must be equal to or less than the Queue Buffer Max Count (" + bufferMaxMessages + ")").build());
- }
-
- return errors;
- }
@Override
public Set<Relationship> getRelationships() {
@@ -298,71 +285,131 @@ public class PutKafka extends AbstractProcessor {
return relationships;
}
- @OnStopped
- public void closeProducers() {
- Producer<byte[], byte[]> producer;
+ @Override
+ protected Collection<ValidationResult> customValidate(final ValidationContext validationContext) {
+ final List<ValidationResult> results = new ArrayList<>();
+
+ final String partitionStrategy = validationContext.getProperty(PARTITION_STRATEGY).getValue();
+ if (partitionStrategy.equalsIgnoreCase(USER_DEFINED_PARTITIONING.getValue()) && !validationContext.getProperty(PARTITION).isSet()) {
+ results.add(new ValidationResult.Builder().subject("Partition").valid(false).explanation(
+ "The <Partition> property must be set when configured to use the User-Defined Partitioning Strategy").build());
+ }
+
+ return results;
+ }
+
+ protected Producer<byte[], byte[]> getProducer() {
+ return producer;
+ }
- while ((producer = producers.poll()) != null) {
+ @OnStopped
+ public void cleanup() {
+ final Producer<byte[], byte[]> producer = getProducer();
+ if (producer != null) {
producer.close();
}
+
+ for (final FlowFileMessageBatch batch : activeBatches) {
+ batch.cancelOrComplete();
+ }
+ }
+
+ @OnScheduled
+ public void createProducer(final ProcessContext context) {
+ producer = new KafkaProducer<byte[], byte[]>(createConfig(context), new ByteArraySerializer(), new ByteArraySerializer());
}
- protected ProducerConfig createConfig(final ProcessContext context) {
+ protected int getActiveMessageBatchCount() {
+ return activeBatches.size();
+ }
+
+ protected int getCompleteMessageBatchCount() {
+ return completeBatches.size();
+ }
+
+ protected Properties createConfig(final ProcessContext context) {
final String brokers = context.getProperty(SEED_BROKERS).getValue();
final Properties properties = new Properties();
- properties.setProperty("metadata.broker.list", brokers);
- properties.setProperty("request.required.acks", context.getProperty(DELIVERY_GUARANTEE).getValue());
+ properties.setProperty("bootstrap.servers", brokers);
+ properties.setProperty("acks", context.getProperty(DELIVERY_GUARANTEE).getValue());
properties.setProperty("client.id", context.getProperty(CLIENT_NAME).getValue());
- properties.setProperty("request.timeout.ms", String.valueOf(context.getProperty(TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).longValue()));
- properties.setProperty("message.send.max.retries", "1");
- properties.setProperty("producer.type", context.getProperty(PRODUCER_TYPE).getValue());
- properties.setProperty("batch.num.messages", context.getProperty(BATCH_NUM_MESSAGES).getValue());
+ final String timeout = String.valueOf(context.getProperty(TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).longValue());
+ properties.setProperty("timeout.ms", timeout);
+ properties.setProperty("metadata.fetch.timeout.ms", timeout);
- final Long queueBufferingMillis = context.getProperty(QUEUE_BUFFERING_MAX).asTimePeriod(TimeUnit.MILLISECONDS);
- if (queueBufferingMillis != null) {
- properties.setProperty("queue.buffering.max.ms", String.valueOf(queueBufferingMillis));
- }
- properties.setProperty("queue.buffering.max.messages", context.getProperty(QUEUE_BUFFERING_MAX_MESSAGES).getValue());
+ properties.setProperty("batch.size", context.getProperty(BATCH_NUM_MESSAGES).getValue());
+ properties.setProperty("max.request.size", String.valueOf(context.getProperty(MAX_RECORD_SIZE).asDataSize(DataUnit.B).longValue()));
- final Long queueEnqueueTimeoutMillis = context.getProperty(QUEUE_ENQUEUE_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS);
- if (queueEnqueueTimeoutMillis != null) {
- properties.setProperty("queue.enqueue.timeout.ms", String.valueOf(queueEnqueueTimeoutMillis));
- }
+ final long maxBufferSize = context.getProperty(MAX_BUFFER_SIZE).asDataSize(DataUnit.B).longValue();
+ properties.setProperty("buffer.memory", String.valueOf(maxBufferSize));
final String compressionCodec = context.getProperty(COMPRESSION_CODEC).getValue();
- properties.setProperty("compression.codec", compressionCodec);
+ properties.setProperty("compression.type", compressionCodec);
- final String compressedTopics = context.getProperty(COMPRESSED_TOPICS).getValue();
- if (compressedTopics != null) {
- properties.setProperty("compressed.topics", compressedTopics);
+ final Long queueBufferingMillis = context.getProperty(QUEUE_BUFFERING_MAX).asTimePeriod(TimeUnit.MILLISECONDS);
+ if (queueBufferingMillis != null) {
+ properties.setProperty("linger.ms", String.valueOf(queueBufferingMillis));
}
- return new ProducerConfig(properties);
- }
+ properties.setProperty("retries", "0");
+ properties.setProperty("block.on.buffer.full", "false");
- protected Producer<byte[], byte[]> createProducer(final ProcessContext context) {
- return new Producer<>(createConfig(context));
+ return properties;
}
- private Producer<byte[], byte[]> borrowProducer(final ProcessContext context) {
- final Producer<byte[], byte[]> producer = producers.poll();
- return producer == null ? createProducer(context) : producer;
- }
+ private Integer getPartition(final ProcessContext context, final FlowFile flowFile, final String topic) {
+ final long unnormalizedIndex;
+
+ final String partitionStrategy = context.getProperty(PARTITION_STRATEGY).getValue();
+ if (partitionStrategy.equalsIgnoreCase(ROUND_ROBIN_PARTITIONING.getValue())) {
+ AtomicLong partitionIndex = partitionIndexMap.get(topic);
+ if (partitionIndex == null) {
+ partitionIndex = new AtomicLong(0L);
+ final AtomicLong existing = partitionIndexMap.putIfAbsent(topic, partitionIndex);
+ if (existing != null) {
+ partitionIndex = existing;
+ }
+ }
+
+ unnormalizedIndex = partitionIndex.getAndIncrement();
+ } else if (partitionStrategy.equalsIgnoreCase(RANDOM_PARTITIONING.getValue())) {
+ return null;
+ } else {
+ if (context.getProperty(PARTITION).isSet()) {
+ final String partitionValue = context.getProperty(PARTITION).evaluateAttributeExpressions(flowFile).getValue();
+
+ if (NUMBER_PATTERN.matcher(partitionValue).matches()) {
+ // Subtract 1 because if the partition is "3" then we want to get index 2 into the List of partitions.
+ unnormalizedIndex = Long.parseLong(partitionValue) - 1;
+ } else {
+ unnormalizedIndex = partitionValue.hashCode();
+ }
+ } else {
+ return null;
+ }
+ }
- private void returnProducer(final Producer<byte[], byte[]> producer) {
- producers.offer(producer);
+ final Producer<byte[], byte[]> producer = getProducer();
+ final List<PartitionInfo> partitionInfos = producer.partitionsFor(topic);
+ final int partitionIdx = (int) (unnormalizedIndex % partitionInfos.size());
+ return partitionInfos.get(partitionIdx).partition();
}
@Override
- public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
+ public void onTrigger(final ProcessContext context, final ProcessSessionFactory sessionFactory) throws ProcessException {
+ FlowFileMessageBatch batch;
+ while ((batch = completeBatches.poll()) != null) {
+ batch.completeSession();
+ }
+
+ final ProcessSession session = sessionFactory.createSession();
final FlowFile flowFile = session.get();
if (flowFile == null) {
return;
}
- final long start = System.nanoTime();
final String topic = context.getProperty(TOPIC).evaluateAttributeExpressions(flowFile).getValue();
final String key = context.getProperty(KEY).evaluateAttributeExpressions(flowFile).getValue();
final byte[] keyBytes = key == null ? null : key.getBytes(StandardCharsets.UTF_8);
@@ -371,8 +418,7 @@ public class PutKafka extends AbstractProcessor {
delimiter = delimiter.replace("\\n", "\n").replace("\\r", "\r").replace("\\t", "\t");
}
- final long maxBufferSize = context.getProperty(MAX_BUFFER_SIZE).asDataSize(DataUnit.B).longValue();
- final Producer<byte[], byte[]> producer = borrowProducer(context);
+ final Producer<byte[], byte[]> producer = getProducer();
if (delimiter == null) {
// Send the entire FlowFile as a single message.
@@ -384,31 +430,38 @@ public class PutKafka extends AbstractProcessor {
}
});
- boolean error = false;
+ final Integer partition;
try {
- final KeyedMessage<byte[], byte[]> message;
- if (key == null) {
- message = new KeyedMessage<>(topic, value);
- } else {
- message = new KeyedMessage<>(topic, keyBytes, value);
- }
-
- producer.send(message);
- final long nanos = System.nanoTime() - start;
-
- session.getProvenanceReporter().send(flowFile, "kafka://" + topic);
- session.transfer(flowFile, REL_SUCCESS);
- getLogger().info("Successfully sent {} to Kafka in {} millis", new Object[] { flowFile, TimeUnit.NANOSECONDS.toMillis(nanos) });
+ partition = getPartition(context, flowFile, topic);
} catch (final Exception e) {
- getLogger().error("Failed to send {} to Kafka due to {}; routing to failure", new Object[] { flowFile, e });
+ getLogger().error("Failed to obtain a partition for {} due to {}", new Object[] {flowFile, e});
session.transfer(session.penalize(flowFile), REL_FAILURE);
- error = true;
- } finally {
- if (error) {
- producer.close();
- } else {
- returnProducer(producer);
- }
+ session.commit();
+ return;
+ }
+
+ final ProducerRecord<byte[], byte[]> producerRecord = new ProducerRecord<>(topic, partition, keyBytes, value);
+
+ final FlowFileMessageBatch messageBatch = new FlowFileMessageBatch(session, flowFile, topic);
+ messageBatch.setNumMessages(1);
+ activeBatches.add(messageBatch);
+
+ try {
+ producer.send(producerRecord, new Callback() {
+ @Override
+ public void onCompletion(final RecordMetadata metadata, final Exception exception) {
+ if (exception == null) {
+ // record was successfully sent.
+ messageBatch.addSuccessfulRange(0L, flowFile.getSize(), metadata.offset());
+ } else {
+ messageBatch.addFailedRange(0L, flowFile.getSize(), exception);
+ }
+ }
+ });
+ } catch (final BufferExhaustedException bee) {
+ messageBatch.addFailedRange(0L, flowFile.getSize(), bee);
+ context.yield();
+ return;
}
} else {
final byte[] delimiterBytes = delimiter.getBytes(StandardCharsets.UTF_8);
@@ -418,9 +471,9 @@ public class PutKafka extends AbstractProcessor {
// the stream of bytes in the FlowFile
final NonThreadSafeCircularBuffer buffer = new NonThreadSafeCircularBuffer(delimiterBytes);
- boolean error = false;
- final LongHolder lastMessageOffset = new LongHolder(0L);
final LongHolder messagesSent = new LongHolder(0L);
+ final FlowFileMessageBatch messageBatch = new FlowFileMessageBatch(session, flowFile, topic);
+ activeBatches.add(messageBatch);
try (final ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
session.read(flowFile, new InputStreamCallback() {
@@ -430,13 +483,12 @@ public class PutKafka extends AbstractProcessor {
boolean streamFinished = false;
- final List<KeyedMessage<byte[], byte[]>> messages = new ArrayList<>(); // batch to send
- long messageBytes = 0L; // size of messages in the 'messages' list
-
int nextByte;
try (final InputStream bufferedIn = new BufferedInputStream(rawIn);
final ByteCountingInputStream in = new ByteCountingInputStream(bufferedIn)) {
+ long messageStartOffset = in.getBytesConsumed();
+
// read until we're out of data.
while (!streamFinished) {
nextByte = in.read();
@@ -457,107 +509,309 @@ public class PutKafka extends AbstractProcessor {
}
if (data != null) {
+ final long messageEndOffset = in.getBytesConsumed();
+
// If the message has no data, ignore it.
if (data.length != 0) {
- // either we ran out of data or we reached the end of the message.
- // Either way, create the message because it's ready to send.
- final KeyedMessage<byte[], byte[]> message;
- if (key == null) {
- message = new KeyedMessage<>(topic, data);
- } else {
- message = new KeyedMessage<>(topic, keyBytes, data);
+ final Integer partition;
+ try {
+ partition = getPartition(context, flowFile, topic);
+ } catch (final Exception e) {
+ messageBatch.addFailedRange(messageStartOffset, messageEndOffset, e);
+ getLogger().error("Failed to obtain a partition for {} due to {}", new Object[] {flowFile, e});
+ continue;
}
- // Add the message to the list of messages ready to send. If we've reached our
- // threshold of how many we're willing to send (or if we're out of data), go ahead
- // and send the whole List.
- messages.add(message);
- messageBytes += data.length;
- if (messageBytes >= maxBufferSize || streamFinished) {
- // send the messages, then reset our state.
- try {
- producer.send(messages);
- } catch (final Exception e) {
- // we wrap the general exception in ProcessException because we want to separate
- // failures in sending messages from general Exceptions that would indicate bugs
- // in the Processor. Failure to send a message should be handled appropriately, but
- // we don't want to catch the general Exception or RuntimeException in order to catch
- // failures from Kafka's Producer.
- throw new ProcessException("Failed to send messages to Kafka", e);
- }
-
- messagesSent.addAndGet(messages.size()); // count number of messages sent
-
- // reset state
- messages.clear();
- messageBytes = 0;
-
- // We've successfully sent a batch of messages. Keep track of the byte offset in the
- // FlowFile of the last successfully sent message. This way, if the messages cannot
- // all be successfully sent, we know where to split off the data. This allows us to then
- // split off the first X number of bytes and send to 'success' and then split off the rest
- // and send them to 'failure'.
- lastMessageOffset.set(in.getBytesConsumed());
+
+ final ProducerRecord<byte[], byte[]> producerRecord = new ProducerRecord<>(topic, partition, keyBytes, data);
+ final long rangeStart = messageStartOffset;
+
+ try {
+ producer.send(producerRecord, new Callback() {
+ @Override
+ public void onCompletion(final RecordMetadata metadata, final Exception exception) {
+ if (exception == null) {
+ // record was successfully sent.
+ messageBatch.addSuccessfulRange(rangeStart, messageEndOffset, metadata.offset());
+ } else {
+ messageBatch.addFailedRange(rangeStart, messageEndOffset, exception);
+ }
+ }
+ });
+
+ messagesSent.incrementAndGet();
+ } catch (final BufferExhaustedException bee) {
+ // Not enough room in the buffer. Add from the beginning of this message to end of FlowFile as a failed range
+ messageBatch.addFailedRange(messageStartOffset, flowFile.getSize(), bee);
+ context.yield();
+ return;
}
+
}
+
// reset BAOS so that we can start a new message.
baos.reset();
data = null;
-
- }
- }
-
- // If there are messages left, send them
- if (!messages.isEmpty()) {
- try {
- messagesSent.addAndGet(messages.size()); // add count of messages
- producer.send(messages);
- } catch (final Exception e) {
- throw new ProcessException("Failed to send messages to Kafka", e);
+ messageStartOffset = in.getBytesConsumed();
}
}
}
}
});
- final long nanos = System.nanoTime() - start;
- session.getProvenanceReporter().send(flowFile, "kafka://" + topic, "Sent " + messagesSent.get() + " messages");
- session.transfer(flowFile, REL_SUCCESS);
- getLogger().info("Successfully sent {} messages to Kafka for {} in {} millis", new Object[] { messagesSent.get(), flowFile, TimeUnit.NANOSECONDS.toMillis(nanos) });
- } catch (final ProcessException pe) {
- error = true;
-
- // There was a failure sending messages to Kafka. Iff the lastMessageOffset is 0, then all of them failed and we can
- // just route the FlowFile to failure. Otherwise, some messages were successful, so split them off and send them to
- // 'success' while we send the others to 'failure'.
- final long offset = lastMessageOffset.get();
- if (offset == 0L) {
- // all of the messages failed to send. Route FlowFile to failure
- getLogger().error("Failed to send {} to Kafka due to {}; routing to fialure", new Object[] { flowFile, pe.getCause() });
- session.transfer(session.penalize(flowFile), REL_FAILURE);
+ messageBatch.setNumMessages(messagesSent.get());
+ }
+ }
+ }
+
+
+ private static class Range {
+ private final long start;
+ private final long end;
+ private final Long kafkaOffset;
+
+ public Range(final long start, final long end, final Long kafkaOffset) {
+ this.start = start;
+ this.end = end;
+ this.kafkaOffset = kafkaOffset;
+ }
+
+ public long getStart() {
+ return start;
+ }
+
+ public long getEnd() {
+ return end;
+ }
+
+ public Long getKafkaOffset() {
+ return kafkaOffset;
+ }
+
+ @Override
+ public String toString() {
+ return "Range[" + start + "-" + end + "]";
+ }
+ }
+
+ private class FlowFileMessageBatch {
+ private final ProcessSession session;
+ private final FlowFile flowFile;
+ private final String topic;
+ private final long startTime = System.nanoTime();
+
+ private final List<Range> successfulRanges = new ArrayList<>();
+ private final List<Range> failedRanges = new ArrayList<>();
+
+ private Exception lastFailureReason;
+ private long numMessages = -1L;
+ private long completeTime = 0L;
+ private boolean canceled = false;
+
+ public FlowFileMessageBatch(final ProcessSession session, final FlowFile flowFile, final String topic) {
+ this.session = session;
+ this.flowFile = flowFile;
+ this.topic = topic;
+ }
+
+ public synchronized void cancelOrComplete() {
+ if (isComplete()) {
+ completeSession();
+ return;
+ }
+
+ this.canceled = true;
+
+ session.rollback();
+ successfulRanges.clear();
+ failedRanges.clear();
+ }
+
+ public synchronized void addSuccessfulRange(final long start, final long end, final long kafkaOffset) {
+ if (canceled) {
+ return;
+ }
+
+ successfulRanges.add(new Range(start, end, kafkaOffset));
+
+ if (isComplete()) {
+ activeBatches.remove(this);
+ completeBatches.add(this);
+ completeTime = System.nanoTime();
+ }
+ }
+
+ public synchronized void addFailedRange(final long start, final long end, final Exception e) {
+ if (canceled) {
+ return;
+ }
+
+ failedRanges.add(new Range(start, end, null));
+ lastFailureReason = e;
+
+ if (isComplete()) {
+ activeBatches.remove(this);
+ completeBatches.add(this);
+ completeTime = System.nanoTime();
+ }
+ }
+
+ private boolean isComplete() {
+ return !canceled && (numMessages > -1) && (successfulRanges.size() + failedRanges.size() >= numMessages);
+ }
+
+ public synchronized void setNumMessages(final long msgCount) {
+ this.numMessages = msgCount;
+
+ if (isComplete()) {
+ activeBatches.remove(this);
+ completeBatches.add(this);
+ completeTime = System.nanoTime();
+ }
+ }
+
+ private Long getMin(final Long a, final Long b) {
+ if (a == null && b == null) {
+ return null;
+ }
+
+ if (a == null) {
+ return b;
+ }
+
+ if (b == null) {
+ return a;
+ }
+
+ return Math.min(a, b);
+ }
+
+ private Long getMax(final Long a, final Long b) {
+ if (a == null && b == null) {
+ return null;
+ }
+
+ if (a == null) {
+ return b;
+ }
+
+ if (b == null) {
+ return a;
+ }
+
+ return Math.max(a, b);
+ }
+
+ private void transferRanges(final List<Range> ranges, final Relationship relationship) {
+ Collections.sort(ranges, new Comparator<Range>() {
+ @Override
+ public int compare(final Range o1, final Range o2) {
+ return Long.compare(o1.getStart(), o2.getStart());
+ }
+ });
+
+ for (int i = 0; i < ranges.size(); i++) {
+ Range range = ranges.get(i);
+ int count = 1;
+ Long smallestKafkaOffset = range.getKafkaOffset();
+ Long largestKafkaOffset = range.getKafkaOffset();
+
+ while (i + 1 < ranges.size()) {
+ // Check if the next range in the List continues where this one left off.
+ final Range nextRange = ranges.get(i + 1);
+
+ if (nextRange.getStart() == range.getEnd()) {
+ // We have two ranges in a row that are contiguous; combine them into a single Range.
+ range = new Range(range.getStart(), nextRange.getEnd(), null);
+
+ smallestKafkaOffset = getMin(smallestKafkaOffset, nextRange.getKafkaOffset());
+ largestKafkaOffset = getMax(largestKafkaOffset, nextRange.getKafkaOffset());
+ count++;
+ i++;
+ } else {
+ break;
+ }
+ }
+
+ // Create a FlowFile for this range.
+ FlowFile child = session.clone(flowFile, range.getStart(), range.getEnd() - range.getStart());
+ if (relationship == REL_SUCCESS) {
+ session.getProvenanceReporter().send(child, getTransitUri(), "Sent " + count + " messages; Kafka offsets range from " + smallestKafkaOffset + " to " + largestKafkaOffset);
+ session.transfer(child, relationship);
} else {
- // Some of the messages were sent successfully. We want to split off the successful messages from the failed messages.
- final FlowFile successfulMessages = session.clone(flowFile, 0L, offset);
- final FlowFile failedMessages = session.clone(flowFile, offset, flowFile.getSize() - offset);
-
- getLogger().error("Successfully sent {} of the messages from {} but then failed to send the rest. Original FlowFile split into"
- + " two: {} routed to 'success', {} routed to 'failure'. Failure was due to {}", new Object[] {
- messagesSent.get(), flowFile, successfulMessages, failedMessages, pe.getCause() });
-
- session.transfer(successfulMessages, REL_SUCCESS);
- session.transfer(session.penalize(failedMessages), REL_FAILURE);
- session.remove(flowFile);
- session.getProvenanceReporter().send(successfulMessages, "kafka://" + topic);
+ session.transfer(session.penalize(child), relationship);
}
- } finally {
- if (error) {
- producer.close();
+ }
+ }
+
+ private String getTransitUri() {
+ final List<PartitionInfo> partitions = getProducer().partitionsFor(topic);
+ if (partitions.isEmpty()) {
+ return "kafka://unknown-host" + "/topics/" + topic;
+ }
+
+ final PartitionInfo info = partitions.get(0);
+ final Node leader = info.leader();
+ final String host = leader.host();
+ final int port = leader.port();
+
+ return "kafka://" + host + ":" + port + "/topics/" + topic;
+ }
+
+ public synchronized void completeSession() {
+ if (canceled) {
+ return;
+ }
+
+ if (successfulRanges.isEmpty() && failedRanges.isEmpty()) {
+ getLogger().info("Completed processing {} but sent 0 FlowFiles to Kafka", new Object[] {flowFile});
+ session.transfer(flowFile, REL_SUCCESS);
+ session.commit();
+ return;
+ }
+
+ if (successfulRanges.isEmpty()) {
+ getLogger().error("Failed to send {} to Kafka; routing to 'failure'; last failure reason reported was {};", new Object[] {flowFile, lastFailureReason});
+ session.transfer(session.penalize(flowFile), REL_FAILURE);
+ session.commit();
+ return;
+ }
+
+ if (failedRanges.isEmpty()) {
+ final long transferMillis = TimeUnit.NANOSECONDS.toMillis(completeTime - startTime);
+
+ if (successfulRanges.size() == 1) {
+ final Long kafkaOffset = successfulRanges.get(0).getKafkaOffset();
+ final String msg = "Sent 1 message" + ((kafkaOffset == null) ? "" : ("; Kafka offset = " + kafkaOffset));
+ session.getProvenanceReporter().send(flowFile, getTransitUri(), msg);
} else {
- returnProducer(producer);
+ long smallestKafkaOffset = successfulRanges.get(0).getKafkaOffset();
+ long largestKafkaOffset = successfulRanges.get(0).getKafkaOffset();
+
+ for (final Range range : successfulRanges) {
+ smallestKafkaOffset = Math.min(smallestKafkaOffset, range.getKafkaOffset());
+ largestKafkaOffset = Math.max(largestKafkaOffset, range.getKafkaOffset());
+ }
+
+ session.getProvenanceReporter().send(flowFile, getTransitUri(),
+ "Sent " + successfulRanges.size() + " messages; Kafka offsets range from " + smallestKafkaOffset + " to " + largestKafkaOffset);
}
+
+ session.transfer(flowFile, REL_SUCCESS);
+ getLogger().info("Successfully sent {} messages to Kafka for {} in {} millis", new Object[] {successfulRanges.size(), flowFile, transferMillis});
+ session.commit();
+ return;
}
+ // At this point, the successful ranges is not empty and the failed ranges is not empty. This indicates that some messages made their way to Kafka
+ // successfully and some failed. We will address this by splitting apart the source FlowFile into children and sending the successful messages to 'success'
+ // and the failed messages to 'failure'.
+ transferRanges(successfulRanges, REL_SUCCESS);
+ transferRanges(failedRanges, REL_FAILURE);
+ session.remove(flowFile);
+ getLogger().error("Successfully sent {} messages to Kafka but failed to send {} messages; the last error received was {}",
+ new Object[] {successfulRanges.size(), failedRanges.size(), lastFailureReason});
+ session.commit();
}
}
-
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/22de23ba/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/test/java/org/apache/nifi/processors/kafka/TestPutKafka.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/test/java/org/apache/nifi/processors/kafka/TestPutKafka.java b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/test/java/org/apache/nifi/processors/kafka/TestPutKafka.java
index 750d406..17d1cc8 100644
--- a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/test/java/org/apache/nifi/processors/kafka/TestPutKafka.java
+++ b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/test/java/org/apache/nifi/processors/kafka/TestPutKafka.java
@@ -22,27 +22,33 @@ import static org.junit.Assert.assertTrue;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-
-import kafka.common.FailedToSendMessageException;
-import kafka.javaapi.producer.Producer;
-import kafka.message.CompressionCodec;
-import kafka.producer.KeyedMessage;
-import kafka.producer.ProducerConfig;
-
-import org.apache.nifi.annotation.lifecycle.OnScheduled;
-import org.apache.nifi.processor.ProcessContext;
+import java.util.concurrent.Future;
+
+import org.apache.kafka.clients.producer.BufferExhaustedException;
+import org.apache.kafka.clients.producer.Callback;
+import org.apache.kafka.clients.producer.Producer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.Metric;
+import org.apache.kafka.common.MetricName;
+import org.apache.kafka.common.Node;
+import org.apache.kafka.common.PartitionInfo;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
import org.apache.nifi.provenance.ProvenanceEventRecord;
import org.apache.nifi.provenance.ProvenanceEventType;
import org.apache.nifi.util.MockFlowFile;
import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
-import org.junit.Ignore;
+import org.junit.Assert;
import org.junit.Test;
-import scala.collection.Seq;
+import kafka.common.FailedToSendMessageException;
+
public class TestPutKafka {
@@ -56,24 +62,19 @@ public class TestPutKafka {
runner.setProperty(PutKafka.MESSAGE_DELIMITER, "\\n");
runner.enqueue("Hello World\nGoodbye\n1\n2\n3\n4\n5\n6\n7\n8\n9".getBytes());
- runner.run();
+ runner.run(2); // we have to run twice because the first iteration will result in data being added to a queue in the processor; the second onTrigger call will transfer FlowFiles.
runner.assertAllFlowFilesTransferred(PutKafka.REL_SUCCESS, 1);
- final List<byte[]> messages = proc.getProducer().getMessages();
+ final List<ProducerRecord<byte[], byte[]>> messages = ((MockProducer) proc.getProducer()).getMessages();
assertEquals(11, messages.size());
- assertTrue(Arrays.equals("Hello World".getBytes(StandardCharsets.UTF_8), messages.get(0)));
- assertTrue(Arrays.equals("Goodbye".getBytes(StandardCharsets.UTF_8), messages.get(1)));
- assertTrue(Arrays.equals("1".getBytes(StandardCharsets.UTF_8), messages.get(2)));
- assertTrue(Arrays.equals("2".getBytes(StandardCharsets.UTF_8), messages.get(3)));
- assertTrue(Arrays.equals("3".getBytes(StandardCharsets.UTF_8), messages.get(4)));
- assertTrue(Arrays.equals("4".getBytes(StandardCharsets.UTF_8), messages.get(5)));
- assertTrue(Arrays.equals("5".getBytes(StandardCharsets.UTF_8), messages.get(6)));
- assertTrue(Arrays.equals("6".getBytes(StandardCharsets.UTF_8), messages.get(7)));
- assertTrue(Arrays.equals("7".getBytes(StandardCharsets.UTF_8), messages.get(8)));
- assertTrue(Arrays.equals("8".getBytes(StandardCharsets.UTF_8), messages.get(9)));
- assertTrue(Arrays.equals("9".getBytes(StandardCharsets.UTF_8), messages.get(10)));
+ assertTrue(Arrays.equals("Hello World".getBytes(StandardCharsets.UTF_8), messages.get(0).value()));
+ assertTrue(Arrays.equals("Goodbye".getBytes(StandardCharsets.UTF_8), messages.get(1).value()));
+
+ for (int i = 1; i <= 9; i++) {
+ assertTrue(Arrays.equals(String.valueOf(i).getBytes(StandardCharsets.UTF_8), messages.get(i + 1).value()));
+ }
}
@Test
@@ -87,7 +88,7 @@ public class TestPutKafka {
final String text = "Hello World\nGoodbye\n1\n2\n3\n4\n5\n6\n7\n8\n9";
runner.enqueue(text.getBytes());
- runner.run();
+ runner.run(2);
runner.assertAllFlowFilesTransferred(PutKafka.REL_FAILURE, 1);
final MockFlowFile mff = runner.getFlowFilesForRelationship(PutKafka.REL_FAILURE).get(0);
@@ -96,7 +97,7 @@ public class TestPutKafka {
@Test
public void testPartialFailure() {
- final TestableProcessor proc = new TestableProcessor(2);
+ final TestableProcessor proc = new TestableProcessor(2); // fail after sending 2 messages.
final TestRunner runner = TestRunners.newTestRunner(proc);
runner.setProperty(PutKafka.TOPIC, "topic1");
runner.setProperty(PutKafka.KEY, "key1");
@@ -106,7 +107,7 @@ public class TestPutKafka {
final byte[] bytes = "1\n2\n3\n4".getBytes();
runner.enqueue(bytes);
- runner.run();
+ runner.run(2);
runner.assertTransferCount(PutKafka.REL_SUCCESS, 1);
runner.assertTransferCount(PutKafka.REL_FAILURE, 1);
@@ -119,6 +120,39 @@ public class TestPutKafka {
}
@Test
+ public void testPartialFailureWithSuccessBeforeAndAfter() {
+ final TestableProcessor proc = new TestableProcessor(2, 4); // fail after sending 2 messages, then stop failing after 4
+ final TestRunner runner = TestRunners.newTestRunner(proc);
+ runner.setProperty(PutKafka.TOPIC, "topic1");
+ runner.setProperty(PutKafka.KEY, "key1");
+ runner.setProperty(PutKafka.SEED_BROKERS, "localhost:1234");
+ runner.setProperty(PutKafka.MESSAGE_DELIMITER, "\\n");
+ runner.setProperty(PutKafka.MAX_BUFFER_SIZE, "1 B");
+
+ final byte[] bytes = "1\n2\n3\n4\n5\n6".getBytes();
+ runner.enqueue(bytes);
+ runner.run(2);
+
+ runner.assertTransferCount(PutKafka.REL_SUCCESS, 2);
+ runner.assertTransferCount(PutKafka.REL_FAILURE, 1);
+
+ final List<MockFlowFile> success = runner.getFlowFilesForRelationship(PutKafka.REL_SUCCESS);
+ for (final MockFlowFile successFF : success) {
+ if ('1' == successFF.toByteArray()[0]) {
+ successFF.assertContentEquals("1\n2\n");
+ } else if ('5' == successFF.toByteArray()[0]) {
+ successFF.assertContentEquals("5\n6");
+ } else {
+ Assert.fail("Wrong content for FlowFile; contained " + new String(successFF.toByteArray()));
+ }
+ }
+
+ final MockFlowFile failureFF = runner.getFlowFilesForRelationship(PutKafka.REL_FAILURE).get(0);
+ failureFF.assertContentEquals("3\n4\n");
+ }
+
+
+ @Test
public void testWithEmptyMessages() {
final TestableProcessor proc = new TestableProcessor();
final TestRunner runner = TestRunners.newTestRunner(proc);
@@ -129,16 +163,16 @@ public class TestPutKafka {
final byte[] bytes = "\n\n\n1\n2\n\n\n\n3\n4\n\n\n".getBytes();
runner.enqueue(bytes);
- runner.run();
+ runner.run(2);
runner.assertAllFlowFilesTransferred(PutKafka.REL_SUCCESS, 1);
- final List<byte[]> msgs = proc.getProducer().getMessages();
+ final List<ProducerRecord<byte[], byte[]>> msgs = ((MockProducer) proc.getProducer()).getMessages();
assertEquals(4, msgs.size());
- assertTrue(Arrays.equals("1".getBytes(), msgs.get(0)));
- assertTrue(Arrays.equals("2".getBytes(), msgs.get(1)));
- assertTrue(Arrays.equals("3".getBytes(), msgs.get(2)));
- assertTrue(Arrays.equals("4".getBytes(), msgs.get(3)));
+
+ for (int i = 1; i <= 4; i++) {
+ assertTrue(Arrays.equals(String.valueOf(i).getBytes(), msgs.get(i - 1).value()));
+ }
}
@Test
@@ -154,14 +188,14 @@ public class TestPutKafka {
final byte[] bytes = "\n\n\n1\n2\n\n\n\n3\n4\n\n\n".getBytes();
runner.enqueue(bytes);
- runner.run();
+ runner.run(2);
final List<ProvenanceEventRecord> events = runner.getProvenanceEvents();
assertEquals(1, events.size());
final ProvenanceEventRecord event = events.get(0);
assertEquals(ProvenanceEventType.SEND, event.getEventType());
- assertEquals("kafka://topic1", event.getTransitUri());
- assertEquals("Sent 4 messages", event.getDetails());
+ assertEquals("kafka://localhost:1111/topics/topic1", event.getTransitUri());
+ assertTrue(event.getDetails().startsWith("Sent 4 messages"));
}
@Test
@@ -175,265 +209,270 @@ public class TestPutKafka {
final byte[] bytes = "\n\n\n1\n2\n\n\n\n3\n4\n\n\n".getBytes();
runner.enqueue(bytes);
- runner.run();
+ runner.run(2);
final List<ProvenanceEventRecord> events = runner.getProvenanceEvents();
assertEquals(1, events.size());
final ProvenanceEventRecord event = events.get(0);
assertEquals(ProvenanceEventType.SEND, event.getEventType());
- assertEquals("kafka://topic1", event.getTransitUri());
- }
-
- @Test
- @Ignore("Intended only for local testing; requires an actual running instance of Kafka & ZooKeeper...")
- public void testKeyValuePut() {
- final TestRunner runner = TestRunners.newTestRunner(PutKafka.class);
- runner.setProperty(PutKafka.SEED_BROKERS, "192.168.0.101:9092");
- runner.setProperty(PutKafka.TOPIC, "${kafka.topic}");
- runner.setProperty(PutKafka.KEY, "${kafka.key}");
- runner.setProperty(PutKafka.TIMEOUT, "3 secs");
- runner.setProperty(PutKafka.DELIVERY_GUARANTEE, PutKafka.DELIVERY_REPLICATED.getValue());
-
- keyValuePutExecute(runner);
+ assertEquals("kafka://localhost:1111/topics/topic1", event.getTransitUri());
}
@Test
- @Ignore("Intended only for local testing; requires an actual running instance of Kafka & ZooKeeper...")
- public void testKeyValuePutAsync() {
- final TestRunner runner = TestRunners.newTestRunner(PutKafka.class);
- runner.setProperty(PutKafka.SEED_BROKERS, "192.168.0.101:9092");
- runner.setProperty(PutKafka.TOPIC, "${kafka.topic}");
- runner.setProperty(PutKafka.KEY, "${kafka.key}");
- runner.setProperty(PutKafka.TIMEOUT, "3 secs");
- runner.setProperty(PutKafka.PRODUCER_TYPE, PutKafka.PRODUCTER_TYPE_ASYNCHRONOUS.getValue());
- runner.setProperty(PutKafka.DELIVERY_GUARANTEE, PutKafka.DELIVERY_REPLICATED.getValue());
-
- keyValuePutExecute(runner);
- }
+ public void testRoundRobinAcrossMultipleMessages() {
+ final TestableProcessor proc = new TestableProcessor();
- private void keyValuePutExecute(final TestRunner runner) {
- final Map<String, String> attributes = new HashMap<>();
- attributes.put("kafka.topic", "test");
- attributes.put("kafka.key", "key3");
+ final TestRunner runner = TestRunners.newTestRunner(proc);
+ runner.setProperty(PutKafka.TOPIC, "topic1");
+ runner.setProperty(PutKafka.KEY, "key1");
+ runner.setProperty(PutKafka.SEED_BROKERS, "localhost:1234");
+ runner.setProperty(PutKafka.PARTITION_STRATEGY, PutKafka.ROUND_ROBIN_PARTITIONING);
- final byte[] data = "Hello, World, Again! ;)".getBytes();
- runner.enqueue(data, attributes);
- runner.enqueue(data, attributes);
- runner.enqueue(data, attributes);
- runner.enqueue(data, attributes);
+ runner.enqueue("hello".getBytes());
+ runner.enqueue("there".getBytes());
+ runner.enqueue("how are you".getBytes());
+ runner.enqueue("today".getBytes());
runner.run(5);
runner.assertAllFlowFilesTransferred(PutKafka.REL_SUCCESS, 4);
- final List<MockFlowFile> mffs = runner.getFlowFilesForRelationship(PutKafka.REL_SUCCESS);
- final MockFlowFile mff = mffs.get(0);
- assertTrue(Arrays.equals(data, mff.toByteArray()));
+ final List<ProducerRecord<byte[], byte[]>> records = ((MockProducer) proc.getProducer()).getMessages();
+ for (int i = 0; i < 3; i++) {
+ assertEquals(i + 1, records.get(i).partition().intValue());
+ }
+
+ assertEquals(1, records.get(3).partition().intValue());
}
@Test
- public void testProducerConfigDefault() {
-
- final TestableProcessor processor = new TestableProcessor();
- final TestRunner runner = TestRunners.newTestRunner(processor);
+ public void testRoundRobinAcrossMultipleMessagesInSameFlowFile() {
+ final TestableProcessor proc = new TestableProcessor();
+ final TestRunner runner = TestRunners.newTestRunner(proc);
runner.setProperty(PutKafka.TOPIC, "topic1");
runner.setProperty(PutKafka.KEY, "key1");
runner.setProperty(PutKafka.SEED_BROKERS, "localhost:1234");
- runner.setProperty(PutKafka.MESSAGE_DELIMITER, "\\n");
+ runner.setProperty(PutKafka.PARTITION_STRATEGY, PutKafka.ROUND_ROBIN_PARTITIONING);
+ runner.setProperty(PutKafka.MESSAGE_DELIMITER, "\n");
- final ProcessContext context = runner.getProcessContext();
- final ProducerConfig config = processor.createConfig(context);
+ runner.enqueue("hello\nthere\nhow are you\ntoday".getBytes());
- // Check the codec
- final CompressionCodec codec = config.compressionCodec();
- assertTrue(codec instanceof kafka.message.NoCompressionCodec$);
+ runner.run(2);
- // Check compressed topics
- final Seq<String> compressedTopics = config.compressedTopics();
- assertEquals(0, compressedTopics.size());
+ runner.assertAllFlowFilesTransferred(PutKafka.REL_SUCCESS, 1);
- // Check the producer type
- final String actualProducerType = config.producerType();
- assertEquals(PutKafka.PRODUCER_TYPE.getDefaultValue(), actualProducerType);
+ final List<ProducerRecord<byte[], byte[]>> records = ((MockProducer) proc.getProducer()).getMessages();
+ for (int i = 0; i < 3; i++) {
+ assertEquals(i + 1, records.get(i).partition().intValue());
+ }
+ assertEquals(1, records.get(3).partition().intValue());
}
- @Test
- public void testProducerConfigAsyncWithCompression() {
- final TestableProcessor processor = new TestableProcessor();
- final TestRunner runner = TestRunners.newTestRunner(processor);
+ @Test
+ public void testUserDefinedPartition() {
+ final TestableProcessor proc = new TestableProcessor();
+ final TestRunner runner = TestRunners.newTestRunner(proc);
runner.setProperty(PutKafka.TOPIC, "topic1");
runner.setProperty(PutKafka.KEY, "key1");
runner.setProperty(PutKafka.SEED_BROKERS, "localhost:1234");
- runner.setProperty(PutKafka.MESSAGE_DELIMITER, "\\n");
- runner.setProperty(PutKafka.PRODUCER_TYPE, PutKafka.PRODUCTER_TYPE_ASYNCHRONOUS.getValue());
- runner.setProperty(PutKafka.COMPRESSION_CODEC, PutKafka.COMPRESSION_CODEC_SNAPPY.getValue());
- runner.setProperty(PutKafka.COMPRESSED_TOPICS, "topic01,topic02,topic03");
+ runner.setProperty(PutKafka.PARTITION_STRATEGY, PutKafka.USER_DEFINED_PARTITIONING);
+ runner.setProperty(PutKafka.PARTITION, "${part}");
+ runner.setProperty(PutKafka.MESSAGE_DELIMITER, "\n");
- final ProcessContext context = runner.getProcessContext();
- final ProducerConfig config = processor.createConfig(context);
+ final Map<String, String> attrs = new HashMap<>();
+ attrs.put("part", "3");
+ runner.enqueue("hello\nthere\nhow are you\ntoday".getBytes(), attrs);
- // Check that the codec is snappy
- final CompressionCodec codec = config.compressionCodec();
- assertTrue(codec instanceof kafka.message.SnappyCompressionCodec$);
+ runner.run(2);
- // Check compressed topics
- final Seq<String> compressedTopics = config.compressedTopics();
- assertEquals(3, compressedTopics.size());
- assertTrue(compressedTopics.contains("topic01"));
- assertTrue(compressedTopics.contains("topic02"));
- assertTrue(compressedTopics.contains("topic03"));
-
- // Check the producer type
- final String actualProducerType = config.producerType();
- assertEquals("async", actualProducerType);
+ runner.assertAllFlowFilesTransferred(PutKafka.REL_SUCCESS, 1);
+ final List<ProducerRecord<byte[], byte[]>> records = ((MockProducer) proc.getProducer()).getMessages();
+ for (int i = 0; i < 4; i++) {
+ assertEquals(3, records.get(i).partition().intValue());
+ }
}
- @Test
- public void testProducerConfigAsyncQueueThresholds() {
- final TestableProcessor processor = new TestableProcessor();
- final TestRunner runner = TestRunners.newTestRunner(processor);
+ @Test
+ public void testUserDefinedPartitionWithInvalidValue() {
+ final TestableProcessor proc = new TestableProcessor();
+
+ final TestRunner runner = TestRunners.newTestRunner(proc);
runner.setProperty(PutKafka.TOPIC, "topic1");
runner.setProperty(PutKafka.KEY, "key1");
runner.setProperty(PutKafka.SEED_BROKERS, "localhost:1234");
- runner.setProperty(PutKafka.MESSAGE_DELIMITER, "\\n");
- runner.setProperty(PutKafka.PRODUCER_TYPE, PutKafka.PRODUCTER_TYPE_ASYNCHRONOUS.getValue());
- runner.setProperty(PutKafka.QUEUE_BUFFERING_MAX, "7 secs");
- runner.setProperty(PutKafka.QUEUE_BUFFERING_MAX_MESSAGES, "535");
- runner.setProperty(PutKafka.QUEUE_ENQUEUE_TIMEOUT, "200 ms");
+ runner.setProperty(PutKafka.PARTITION_STRATEGY, PutKafka.USER_DEFINED_PARTITIONING);
+ runner.setProperty(PutKafka.PARTITION, "${part}");
+ runner.setProperty(PutKafka.MESSAGE_DELIMITER, "\n");
- final ProcessContext context = runner.getProcessContext();
- final ProducerConfig config = processor.createConfig(context);
+ final Map<String, String> attrs = new HashMap<>();
+ attrs.put("part", "bogus");
+ runner.enqueue("hello\nthere\nhow are you\ntoday".getBytes(), attrs);
- // Check that the queue thresholds were properly translated
- assertEquals(7000, config.queueBufferingMaxMs());
- assertEquals(535, config.queueBufferingMaxMessages());
- assertEquals(200, config.queueEnqueueTimeoutMs());
+ runner.run(2);
- // Check the producer type
- final String actualProducerType = config.producerType();
- assertEquals("async", actualProducerType);
-
- }
-
- @Test
- public void testProducerConfigInvalidBatchSize() {
-
- final TestableProcessor processor = new TestableProcessor();
- final TestRunner runner = TestRunners.newTestRunner(processor);
-
- runner.setProperty(PutKafka.TOPIC, "topic1");
- runner.setProperty(PutKafka.KEY, "key1");
- runner.setProperty(PutKafka.SEED_BROKERS, "localhost:1234");
- runner.setProperty(PutKafka.MESSAGE_DELIMITER, "\\n");
- runner.setProperty(PutKafka.PRODUCER_TYPE, PutKafka.PRODUCTER_TYPE_ASYNCHRONOUS.getValue());
- runner.setProperty(PutKafka.BATCH_NUM_MESSAGES, "200");
- runner.setProperty(PutKafka.QUEUE_BUFFERING_MAX_MESSAGES, "100");
+ runner.assertAllFlowFilesTransferred(PutKafka.REL_SUCCESS, 1);
- runner.assertNotValid();
+ final List<ProducerRecord<byte[], byte[]>> records = ((MockProducer) proc.getProducer()).getMessages();
+ // should all be the same partition, regardless of what partition it is.
+ final int partition = records.get(0).partition().intValue();
+ for (int i = 0; i < 4; i++) {
+ assertEquals(partition, records.get(i).partition().intValue());
+ }
}
- @Test
- public void testProducerConfigAsyncDefaultEnqueueTimeout() {
- final TestableProcessor processor = new TestableProcessor();
- final TestRunner runner = TestRunners.newTestRunner(processor);
+ @Test
+ public void testFullBuffer() {
+ final TestableProcessor proc = new TestableProcessor();
+ final TestRunner runner = TestRunners.newTestRunner(proc);
runner.setProperty(PutKafka.TOPIC, "topic1");
runner.setProperty(PutKafka.KEY, "key1");
runner.setProperty(PutKafka.SEED_BROKERS, "localhost:1234");
- runner.setProperty(PutKafka.MESSAGE_DELIMITER, "\\n");
- runner.setProperty(PutKafka.PRODUCER_TYPE, PutKafka.PRODUCTER_TYPE_ASYNCHRONOUS.getValue());
- // Do not set QUEUE_ENQUEUE_TIMEOUT
+ runner.setProperty(PutKafka.MESSAGE_DELIMITER, "\n");
+ runner.setProperty(PutKafka.MAX_BUFFER_SIZE, "5 B");
+ proc.setMaxQueueSize(10L); // will take 4 bytes for key and 1 byte for value.
- final ProcessContext context = runner.getProcessContext();
- final ProducerConfig config = processor.createConfig(context);
+ runner.enqueue("1\n2\n3\n4\n".getBytes());
+ runner.run(2);
- // Check that the enqueue timeout defaults to -1
- assertEquals(-1, config.queueEnqueueTimeoutMs());
-
- // Check the producer type
- final String actualProducerType = config.producerType();
- assertEquals("async", actualProducerType);
+ runner.assertTransferCount(PutKafka.REL_SUCCESS, 1);
+ runner.assertTransferCount(PutKafka.REL_FAILURE, 1);
+ runner.getFlowFilesForRelationship(PutKafka.REL_SUCCESS).get(0).assertContentEquals("1\n2\n");
+ runner.getFlowFilesForRelationship(PutKafka.REL_FAILURE).get(0).assertContentEquals("3\n4\n");
}
- private static class TestableProcessor extends PutKafka {
- private MockProducer producer;
- private int failAfter = Integer.MAX_VALUE;
+ /**
+ * Used to override the {@link #getProducer()} method so that we can enforce that our MockProducer is used
+ */
+ private static class TestableProcessor extends PutKafka {
+ private final MockProducer producer;
public TestableProcessor() {
+ this(null);
}
- public TestableProcessor(final int failAfter) {
- this.failAfter = failAfter;
+ public TestableProcessor(final Integer failAfter) {
+ this(failAfter, null);
}
- @OnScheduled
- public void instantiateProducer(final ProcessContext context) {
- producer = new MockProducer(createConfig(context));
+ public TestableProcessor(final Integer failAfter, final Integer stopFailingAfter) {
+ producer = new MockProducer();
producer.setFailAfter(failAfter);
+ producer.setStopFailingAfter(stopFailingAfter);
}
@Override
- protected Producer<byte[], byte[]> createProducer(final ProcessContext context) {
- return producer;
- }
-
- public MockProducer getProducer() {
+ protected Producer<byte[], byte[]> getProducer() {
return producer;
}
- /**
- * Exposed for test verification
- */
- @Override
- public ProducerConfig createConfig(final ProcessContext context) {
- return super.createConfig(context);
+ public void setMaxQueueSize(final long bytes) {
+ producer.setMaxQueueSize(bytes);
}
}
- private static class MockProducer extends Producer<byte[], byte[]> {
+
+ /**
+ * We have our own Mock Producer, which is very similar to the Kafka-supplied one. However, with the Kafka-supplied
+ * Producer, we don't have the ability to tell it to fail after X number of messages; rather, we can only tell it
+ * to fail on the next message. Since we are sending multiple messages in a single onTrigger call for the Processor,
+ * this doesn't allow us to test failure conditions adequately.
+ */
+ private static class MockProducer implements Producer<byte[], byte[]> {
private int sendCount = 0;
- private int failAfter = Integer.MAX_VALUE;
+ private Integer failAfter;
+ private Integer stopFailingAfter;
+ private long queueSize = 0L;
+ private long maxQueueSize = Long.MAX_VALUE;
- private final List<byte[]> messages = new ArrayList<>();
+ private final List<ProducerRecord<byte[], byte[]>> messages = new ArrayList<>();
- public MockProducer(final ProducerConfig config) {
- super(config);
+ public MockProducer() {
+ }
+
+ public void setMaxQueueSize(final long bytes) {
+ this.maxQueueSize = bytes;
+ }
+
+ public List<ProducerRecord<byte[], byte[]>> getMessages() {
+ return messages;
+ }
+
+ public void setFailAfter(final Integer successCount) {
+ failAfter = successCount;
+ }
+
+ public void setStopFailingAfter(final Integer stopFailingAfter) {
+ this.stopFailingAfter = stopFailingAfter;
}
@Override
- public void send(final KeyedMessage<byte[], byte[]> message) {
- if (++sendCount > failAfter) {
- throw new FailedToSendMessageException("Failed to send message", new RuntimeException("Unit test told to fail after " + failAfter + " successful messages"));
+ public Future<RecordMetadata> send(final ProducerRecord<byte[], byte[]> record) {
+ return send(record, null);
+ }
+
+ @Override
+ public Future<RecordMetadata> send(final ProducerRecord<byte[], byte[]> record, final Callback callback) {
+ sendCount++;
+
+ final ByteArraySerializer serializer = new ByteArraySerializer();
+ final int keyBytes = serializer.serialize(record.topic(), record.key()).length;
+ final int valueBytes = serializer.serialize(record.topic(), record.value()).length;
+ if (maxQueueSize - queueSize < keyBytes + valueBytes) {
+ throw new BufferExhaustedException("Queue size is " + queueSize + " but serialized message is " + (keyBytes + valueBytes));
+ }
+
+ queueSize += keyBytes + valueBytes;
+
+ if (failAfter != null && sendCount > failAfter && ((stopFailingAfter == null) || (sendCount < stopFailingAfter + 1))) {
+ final Exception e = new FailedToSendMessageException("Failed to send message", new RuntimeException("Unit test told to fail after " + failAfter + " successful messages"));
+ callback.onCompletion(null, e);
} else {
- messages.add(message.message());
+ messages.add(record);
+ final RecordMetadata meta = new RecordMetadata(new TopicPartition(record.topic(), record.partition() == null ? 1 : record.partition()), 0L, 0L);
+ callback.onCompletion(meta, null);
}
+
+ // we don't actually look at the Future in the processor, so we can just return null
+ return null;
}
- public List<byte[]> getMessages() {
- return messages;
+ @Override
+ public List<PartitionInfo> partitionsFor(String topic) {
+ final Node leader = new Node(1, "localhost", 1111);
+ final Node node2 = new Node(2, "localhost-2", 2222);
+ final Node node3 = new Node(3, "localhost-3", 3333);
+
+ final PartitionInfo partInfo1 = new PartitionInfo(topic, 1, leader, new Node[] {node2, node3}, new Node[0]);
+ final PartitionInfo partInfo2 = new PartitionInfo(topic, 2, leader, new Node[] {node2, node3}, new Node[0]);
+ final PartitionInfo partInfo3 = new PartitionInfo(topic, 3, leader, new Node[] {node2, node3}, new Node[0]);
+
+ final List<PartitionInfo> infos = new ArrayList<>(3);
+ infos.add(partInfo1);
+ infos.add(partInfo2);
+ infos.add(partInfo3);
+ return infos;
}
@Override
- public void send(final List<KeyedMessage<byte[], byte[]>> messages) {
- for (final KeyedMessage<byte[], byte[]> msg : messages) {
- send(msg);
- }
+ public Map<MetricName, ? extends Metric> metrics() {
+ return Collections.emptyMap();
}
- public void setFailAfter(final int successCount) {
- failAfter = successCount;
+ @Override
+ public void close() {
}
}
[26/50] [abbrv] nifi git commit: NIFI-1165: Use FileChannel instead
of RandomAccessFile in order to avoid locking files in Windows
Posted by mc...@apache.org.
NIFI-1165: Use FileChannel instead of RandomAccessFile in order to avoid locking files in Windows
Reviewed by Tony Kurc (tkurc@apache.org)
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/e862f7ff
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/e862f7ff
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/e862f7ff
Branch: refs/heads/NIFI-655
Commit: e862f7ff0393f60ba96afd9c97e756c745f0a19b
Parents: 1e5cc07
Author: Mark Payne <ma...@hotmail.com>
Authored: Mon Nov 16 09:59:59 2015 -0500
Committer: Tony Kurc <tr...@gmail.com>
Committed: Thu Nov 19 01:01:28 2015 -0500
----------------------------------------------------------------------
.../nifi/processors/standard/TailFile.java | 123 +++++++++++++------
.../nifi/processors/standard/TestTailFile.java | 21 ++--
2 files changed, 100 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/e862f7ff/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TailFile.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TailFile.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TailFile.java
index 684e3cc..1c32c00 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TailFile.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TailFile.java
@@ -27,10 +27,12 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
-import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@@ -50,6 +52,7 @@ import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.annotation.lifecycle.OnStopped;
import org.apache.nifi.components.AllowableValue;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.flowfile.FlowFile;
@@ -120,7 +123,7 @@ public class TailFile extends AbstractProcessor {
.description("All FlowFiles are routed to this Relationship.")
.build();
- private volatile TailFileState state = new TailFileState(null, null, null, 0L, 0L, null, new byte[65536]);
+ private volatile TailFileState state = new TailFileState(null, null, null, 0L, 0L, null, ByteBuffer.allocate(65536));
private volatile boolean recoveredRolledFiles = false;
private volatile Long expectedRecoveryChecksum;
private volatile boolean tailFileChanged = false;
@@ -143,7 +146,7 @@ public class TailFile extends AbstractProcessor {
@Override
public void onPropertyModified(final PropertyDescriptor descriptor, final String oldValue, final String newValue) {
if (FILENAME.equals(descriptor)) {
- state = new TailFileState(newValue, null, null, 0L, 0L, null, new byte[65536]);
+ state = new TailFileState(newValue, null, null, 0L, 0L, null, ByteBuffer.allocate(65536));
recoveredRolledFiles = false;
tailFileChanged = true;
} else if (ROLLING_FILENAME_PATTERN.equals(descriptor)) {
@@ -173,7 +176,7 @@ public class TailFile extends AbstractProcessor {
final long timestamp = dis.readLong();
final boolean checksumPresent = dis.readBoolean();
- RandomAccessFile reader = null;
+ FileChannel reader = null;
File tailFile = null;
if (checksumPresent && tailFilename.equals(filename)) {
@@ -198,19 +201,21 @@ public class TailFile extends AbstractProcessor {
// beginning of the file.
getLogger().debug("When recovering state, checksum of tailed file matches the stored checksum. Will resume where left off.");
tailFile = existingTailFile;
- reader = new RandomAccessFile(tailFile, "r");
- reader.seek(position);
+ reader = FileChannel.open(tailFile.toPath(), StandardOpenOption.READ);
+ getLogger().debug("Created RandomAccessFile {} for {} in recoverState", new Object[] {reader, tailFile});
+
+ reader.position(position);
} else {
getLogger().debug("When recovering state, checksum of tailed file does not match the stored checksum. Will begin tailing current file from beginning.");
}
}
}
- state = new TailFileState(tailFilename, tailFile, reader, position, timestamp, checksum, new byte[65536]);
+ state = new TailFileState(tailFilename, tailFile, reader, position, timestamp, checksum, ByteBuffer.allocate(65536));
} else {
expectedRecoveryChecksum = null;
// tailing a new file since the state file was written out. We will reset state.
- state = new TailFileState(tailFilename, null, null, 0L, 0L, null, new byte[65536]);
+ state = new TailFileState(tailFilename, null, null, 0L, 0L, null, ByteBuffer.allocate(65536));
}
getLogger().debug("Recovered state {}", new Object[] {state});
@@ -222,6 +227,30 @@ public class TailFile extends AbstractProcessor {
}
+ @OnStopped
+ public void cleanup() {
+ final TailFileState state = this.state;
+ if (state == null) {
+ return;
+ }
+
+ final FileChannel channel = state.getReader();
+ if (channel == null) {
+ return;
+ }
+
+ try {
+ channel.close();
+ } catch (final IOException ioe) {
+ getLogger().warn("Failed to close file handle during cleanup");
+ }
+
+ getLogger().debug("Closed FileChannel {}", new Object[] {channel});
+
+ this.state = new TailFileState(state.getFilename(), state.getFile(), null, state.getPosition(), state.getTimestamp(), state.getChecksum(), state.getBuffer());
+ }
+
+
public void persistState(final TailFileState state, final String stateFilename) throws IOException {
getLogger().debug("Persisting state {} to {}", new Object[] {state, stateFilename});
@@ -247,23 +276,26 @@ public class TailFile extends AbstractProcessor {
}
}
- private RandomAccessFile createReader(final File file, final long position) {
- final RandomAccessFile reader;
+ private FileChannel createReader(final File file, final long position) {
+ final FileChannel reader;
try {
- reader = new RandomAccessFile(file, "r");
- } catch (final FileNotFoundException fnfe) {
- getLogger().warn("File {} does not exist; will attempt to access file again after the configured Yield Duration has elapsed", new Object[] {file});
+ reader = FileChannel.open(file.toPath(), StandardOpenOption.READ);
+ } catch (final IOException ioe) {
+ getLogger().warn("Could not open {} due to {}; will attempt to access file again after the configured Yield Duration has elapsed", new Object[] {file, ioe});
return null;
}
+ getLogger().debug("Created RandomAccessFile {} for {}", new Object[] {reader, file});
+
try {
- reader.seek(position);
+ reader.position(position);
} catch (final IOException ioe) {
getLogger().error("Failed to read from {} due to {}", new Object[] {file, ioe});
try {
reader.close();
+ getLogger().debug("Closed RandomAccessFile {}", new Object[] {reader});
} catch (final IOException ioe2) {
}
@@ -321,6 +353,7 @@ public class TailFile extends AbstractProcessor {
if (flowFile.getSize() == 0L) {
session.remove(flowFile);
// use a timestamp of lastModified() + 1 so that we do not ingest this file again.
+ cleanup();
state = new TailFileState(context.getProperty(FILENAME).getValue(), null, null, 0L, firstFile.lastModified() + 1L, null, state.getBuffer());
} else {
flowFile = session.putAttribute(flowFile, "filename", firstFile.getName());
@@ -330,6 +363,7 @@ public class TailFile extends AbstractProcessor {
session.transfer(flowFile, REL_SUCCESS);
// use a timestamp of lastModified() + 1 so that we do not ingest this file again.
+ cleanup();
state = new TailFileState(context.getProperty(FILENAME).getValue(), null, null, 0L, firstFile.lastModified() + 1L, null, state.getBuffer());
// must ensure that we do session.commit() before persisting state in order to avoid data loss.
@@ -358,6 +392,7 @@ public class TailFile extends AbstractProcessor {
session.transfer(flowFile, REL_SUCCESS);
// use a timestamp of lastModified() + 1 so that we do not ingest this file again.
+ cleanup();
state = new TailFileState(context.getProperty(FILENAME).getValue(), null, null, 0L, file.lastModified() + 1L, null, state.getBuffer());
// must ensure that we do session.commit() before persisting state in order to avoid data loss.
@@ -382,13 +417,16 @@ public class TailFile extends AbstractProcessor {
if (START_BEGINNING_OF_TIME.getValue().equals(recoverPosition)) {
recoverRolledFiles(context, session);
} else if (START_CURRENT_FILE.getValue().equals(recoverPosition)) {
+ cleanup();
state = new TailFileState(context.getProperty(FILENAME).getValue(), null, null, 0L, 0L, null, state.getBuffer());
} else {
final String filename = context.getProperty(FILENAME).getValue();
final File file = new File(filename);
try {
- final RandomAccessFile raf = new RandomAccessFile(file, "r");
+ final FileChannel channel = FileChannel.open(file.toPath(), StandardOpenOption.READ);
+ getLogger().debug("Created FileChannel {} for {}", new Object[] {channel, file});
+
final Checksum checksum = new CRC32();
final long position = file.length();
final long timestamp = file.lastModified();
@@ -398,8 +436,9 @@ public class TailFile extends AbstractProcessor {
StreamUtils.copy(in, new NullOutputStream(), position);
}
- raf.seek(position);
- state = new TailFileState(filename, file, raf, position, timestamp, checksum, state.getBuffer());
+ channel.position(position);
+ cleanup();
+ state = new TailFileState(filename, file, channel, position, timestamp, checksum, state.getBuffer());
} catch (final IOException ioe) {
getLogger().error("Attempted to position Reader at current position in file {} but failed to do so due to {}", new Object[] {file, ioe.toString()}, ioe);
context.yield();
@@ -419,7 +458,7 @@ public class TailFile extends AbstractProcessor {
// the onTrigger method and then create a new state object after we finish processing the files.
TailFileState state = this.state;
File file = state.getFile();
- RandomAccessFile reader = state.getReader();
+ FileChannel reader = state.getReader();
Checksum checksum = state.getChecksum();
if (checksum == null) {
checksum = new CRC32();
@@ -441,6 +480,7 @@ public class TailFile extends AbstractProcessor {
// Check if file has rotated
long fileLength = file.length();
+ final long lastModified = file.lastModified();
if (fileLength < position) {
// File has rotated. It's possible that it rotated before we finished reading all of the data. As a result, we need
// to check the last rolled-over file and see if it is longer than our position. If so, consume the data past our
@@ -489,6 +529,7 @@ public class TailFile extends AbstractProcessor {
// Since file has rotated, we close the reader, create a new one, and then reset our state.
try {
reader.close();
+ getLogger().debug("Closed RandomAccessFile {}", new Object[] {reader, reader});
} catch (final IOException ioe) {
getLogger().warn("Failed to close reader for {} due to {}", new Object[] {file, ioe});
}
@@ -503,12 +544,12 @@ public class TailFile extends AbstractProcessor {
boolean consumeData = false;
if (fileLength > position) {
consumeData = true;
- } else if (file.lastModified() > timestamp) {
+ } else if (lastModified > timestamp) {
// This can happen if file is truncated, or is replaced with the same amount of data as the old file.
position = 0;
try {
- reader.seek(0L);
+ reader.position(0L);
} catch (final IOException ioe) {
getLogger().error("Failed to seek to beginning of file due to {}", new Object[] {ioe});
context.yield();
@@ -525,7 +566,7 @@ public class TailFile extends AbstractProcessor {
// data has been written to file. Stream it to a new FlowFile.
FlowFile flowFile = session.create();
- final RandomAccessFile fileReader = reader;
+ final FileChannel fileReader = reader;
final LongHolder positionHolder = new LongHolder(position);
flowFile = session.write(flowFile, new OutputStreamCallback() {
@Override
@@ -539,6 +580,7 @@ public class TailFile extends AbstractProcessor {
// If there ended up being no data, just remove the FlowFile
if (flowFile.getSize() == 0) {
session.remove(flowFile);
+ getLogger().debug("No data to consume; removed created FlowFile");
} else {
// determine filename for FlowFile by using <base filename of log file>.<initial offset>-<final offset>.<extension>
final String tailFilename = file.getName();
@@ -559,7 +601,8 @@ public class TailFile extends AbstractProcessor {
TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos));
session.transfer(flowFile, REL_SUCCESS);
position = positionHolder.get();
- timestamp = System.currentTimeMillis();
+ timestamp = lastModified;
+ getLogger().debug("Created {} and routed to success", new Object[] {flowFile});
}
}
@@ -571,6 +614,7 @@ public class TailFile extends AbstractProcessor {
// no data to consume so rather than continually running, yield to allow other processors to use the thread.
// In this case, the state should not have changed, and we will have created no FlowFiles, so we don't have to
// persist the state or commit the session; instead, just return here.
+ getLogger().debug("No data to consume; created no FlowFiles");
context.yield();
return;
}
@@ -598,17 +642,20 @@ public class TailFile extends AbstractProcessor {
* @return The new position after the lines have been read
* @throws java.io.IOException if an I/O error occurs.
*/
- private long readLines(final RandomAccessFile reader, final byte[] buffer, final OutputStream out, final Checksum checksum) throws IOException {
+ private long readLines(final FileChannel reader, final ByteBuffer buffer, final OutputStream out, final Checksum checksum) throws IOException {
try (final ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
- long pos = reader.getFilePointer();
+ long pos = reader.position();
long rePos = pos; // position to re-read
int num;
int linesRead = 0;
boolean seenCR = false;
+ buffer.clear();
while (((num = reader.read(buffer)) != -1)) {
+ buffer.flip();
+
for (int i = 0; i < num; i++) {
- byte ch = buffer[i];
+ byte ch = buffer.get();
switch (ch) {
case '\n':
@@ -647,11 +694,12 @@ public class TailFile extends AbstractProcessor {
}
}
- pos = reader.getFilePointer();
+ pos = reader.position();
}
getLogger().debug("Read {} lines; repositioning reader from {} to {}", new Object[] {linesRead, pos, rePos});
- reader.seek(rePos); // Ensure we can re-read if necessary
+ reader.position(rePos);
+ buffer.clear();
return rePos;
}
}
@@ -726,23 +774,26 @@ public class TailFile extends AbstractProcessor {
/**
- * A simple Java class to hold information about our state so that we can maintain this state across multiple invocations of the Processor
+ * A simple Java class to hold information about our state so that we can maintain this state across multiple invocations of the Processor.
+ *
+ * We use a FileChannel to read from the file, rather than a BufferedInputStream, etc. because we want to be able to read any amount of data
+ * and then reposition the reader if we need to, as a result of a line not being terminated (i.e., no new-line).
*/
static class TailFileState {
private final String filename; // hold onto filename and not just File because we want to match that against the user-defined filename to recover from
private final File file;
- private final RandomAccessFile raf;
+ private final FileChannel fileChannel;
private final long position;
private final long timestamp;
private final Checksum checksum;
- private final byte[] buffer;
+ private final ByteBuffer buffer;
- public TailFileState(final String filename, final File file, final RandomAccessFile raf, final long position, final long timestamp, final Checksum checksum, final byte[] buffer) {
+ public TailFileState(final String filename, final File file, final FileChannel fileChannel, final long position, final long timestamp, final Checksum checksum, final ByteBuffer buffer) {
this.filename = filename;
this.file = file;
- this.raf = raf;
+ this.fileChannel = fileChannel;
this.position = position;
- this.timestamp = (timestamp / 1000) * 1000; // many operating systems will use only second-level precision for last-modified times so cut off milliseconds
+ this.timestamp = timestamp; // many operating systems will use only second-level precision for last-modified times so cut off milliseconds
this.checksum = checksum;
this.buffer = buffer;
}
@@ -755,8 +806,8 @@ public class TailFile extends AbstractProcessor {
return file;
}
- public RandomAccessFile getReader() {
- return raf;
+ public FileChannel getReader() {
+ return fileChannel;
}
public long getPosition() {
@@ -771,7 +822,7 @@ public class TailFile extends AbstractProcessor {
return checksum;
}
- public byte[] getBuffer() {
+ public ByteBuffer getBuffer() {
return buffer;
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/e862f7ff/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTailFile.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTailFile.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTailFile.java
index 85638ad..1445d88 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTailFile.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTailFile.java
@@ -37,6 +37,7 @@ import org.junit.Test;
public class TestTailFile {
private File file;
+ private TailFile processor;
private RandomAccessFile raf;
private TestRunner runner;
@@ -64,7 +65,8 @@ public class TestTailFile {
stateFile.delete();
Assert.assertFalse(stateFile.exists());
- runner = TestRunners.newTestRunner(new TailFile());
+ processor = new TailFile();
+ runner = TestRunners.newTestRunner(processor);
runner.setProperty(TailFile.FILENAME, "target/log.txt");
runner.setProperty(TailFile.STATE_FILE, "target/tail-file.state");
runner.assertValid();
@@ -77,6 +79,8 @@ public class TestTailFile {
if (raf != null) {
raf.close();
}
+
+ processor.cleanup();
}
@@ -153,6 +157,7 @@ public class TestTailFile {
runner.setProperty(TailFile.START_POSITION, TailFile.START_CURRENT_TIME.getValue());
raf.write("hello world\n".getBytes());
+ Thread.sleep(1000);
runner.run(100);
runner.assertAllFlowFilesTransferred(TailFile.REL_SUCCESS, 0);
}
@@ -231,7 +236,9 @@ public class TestTailFile {
raf.write("world".getBytes());
raf.close();
- file.renameTo(new File("target/log1.txt"));
+
+ processor.cleanup(); // Need to do this for Windows because otherwise we cannot rename the file because we have the file open still in the same process.
+ assertTrue(file.renameTo(new File("target/log1.txt")));
raf = new RandomAccessFile(new File("target/log.txt"), "rw");
raf.write("1\n".getBytes());
@@ -255,7 +262,7 @@ public class TestTailFile {
runner.assertAllFlowFilesTransferred(TailFile.REL_SUCCESS, 0);
raf.write("hello\n".getBytes());
- runner.run(1, false, false);
+ runner.run(1, true, false);
runner.assertAllFlowFilesTransferred(TailFile.REL_SUCCESS, 1);
runner.getFlowFilesForRelationship(TailFile.REL_SUCCESS).get(0).assertContentEquals("hello\n");
runner.clearTransferState();
@@ -288,7 +295,7 @@ public class TestTailFile {
runner.assertAllFlowFilesTransferred(TailFile.REL_SUCCESS, 0);
raf.write("hello\n".getBytes());
- runner.run(1, false, false);
+ runner.run(1, true, false);
runner.assertAllFlowFilesTransferred(TailFile.REL_SUCCESS, 1);
runner.getFlowFilesForRelationship(TailFile.REL_SUCCESS).get(0).assertContentEquals("hello\n");
runner.clearTransferState();
@@ -305,6 +312,8 @@ public class TestTailFile {
// write to a new file.
file = new File("target/log.txt");
raf = new RandomAccessFile(file, "rw");
+
+ Thread.sleep(1000L);
raf.write("abc\n".getBytes());
// rename file to log.1
@@ -331,9 +340,6 @@ public class TestTailFile {
runner.run();
runner.assertAllFlowFilesTransferred(TailFile.REL_SUCCESS, 0);
- final long start = System.currentTimeMillis();
- Thread.sleep(1100L);
-
raf.write("Hello, World".getBytes());
runner.run();
runner.assertAllFlowFilesTransferred(TailFile.REL_SUCCESS, 0);
@@ -346,7 +352,6 @@ public class TestTailFile {
assertNotNull(state);
assertEquals("target/log.txt", state.getFilename());
assertTrue(state.getTimestamp() <= System.currentTimeMillis());
- assertTrue(state.getTimestamp() >= start);
assertEquals(14, state.getPosition());
runner.getFlowFilesForRelationship(TailFile.REL_SUCCESS).get(0).assertContentEquals("Hello, World\r\n");
[03/50] [abbrv] nifi git commit: NIFI-748 Fixed logic around handling
partial query results from provenance repository - Ensured that failures
derived form correlating Document to its actual provenance event do fail the
entire query and produce partial r
Posted by mc...@apache.org.
NIFI-748 Fixed logic around handling partial query results from provenance repository
- Ensured that failures derived form correlating Document to its actual provenance event do fail the entire query and produce partial results with warning messages
- Refactored DocsReader.read() operation.
- Added test to validate two conditions where the such failures could occur
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/a4d93c62
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/a4d93c62
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/a4d93c62
Branch: refs/heads/NIFI-655
Commit: a4d93c62c88f594ef0cd3739a0536769f1ab9b26
Parents: 36d00a6
Author: Oleg Zhurakousky <ol...@suitcase.io>
Authored: Fri Nov 13 14:08:39 2015 -0500
Committer: Oleg Zhurakousky <ol...@suitcase.io>
Committed: Fri Nov 13 14:23:31 2015 -0500
----------------------------------------------------------------------
.../PersistentProvenanceRepository.java | 1 +
.../nifi/provenance/lucene/DocsReader.java | 116 ++++++-------------
.../nifi/provenance/lucene/IndexSearch.java | 2 +-
.../nifi/provenance/lucene/LineageQuery.java | 2 +-
.../nifi/provenance/lucene/LuceneUtil.java | 23 ++++
.../TestPersistentProvenanceRepository.java | 70 +++++++++++
6 files changed, 132 insertions(+), 82 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/a4d93c62/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/PersistentProvenanceRepository.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/PersistentProvenanceRepository.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/PersistentProvenanceRepository.java
index 89e1419..1740f51 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/PersistentProvenanceRepository.java
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/PersistentProvenanceRepository.java
@@ -1784,6 +1784,7 @@ public class PersistentProvenanceRepository implements ProvenanceEventRepository
try {
Thread.sleep(100L);
} catch (final InterruptedException ie) {
+ Thread.currentThread().interrupt();
}
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/a4d93c62/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/DocsReader.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/DocsReader.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/DocsReader.java
index c2a7609..7bd800b 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/DocsReader.java
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/DocsReader.java
@@ -17,15 +17,14 @@
package org.apache.nifi.provenance.lucene;
import java.io.File;
-import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
-import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@@ -47,9 +46,6 @@ import org.slf4j.LoggerFactory;
public class DocsReader {
private final Logger logger = LoggerFactory.getLogger(DocsReader.class);
- public DocsReader(final List<File> storageDirectories) {
- }
-
public Set<ProvenanceEventRecord> read(final TopDocs topDocs, final IndexReader indexReader, final Collection<Path> allProvenanceLogFiles,
final AtomicInteger retrievalCount, final int maxResults, final int maxAttributeChars) throws IOException {
if (retrievalCount.get() >= maxResults) {
@@ -100,101 +96,61 @@ public class DocsReader {
}
}
- if ( record == null ) {
- throw new IOException("Failed to find Provenance Event " + d);
- } else {
- return record;
+ if (record == null) {
+ logger.warn("Failed to read Provenance Event for '" + d + "'. The event file may be missing or corrupted");
}
- }
+ return record;
+ }
public Set<ProvenanceEventRecord> read(final List<Document> docs, final Collection<Path> allProvenanceLogFiles,
- final AtomicInteger retrievalCount, final int maxResults, final int maxAttributeChars) throws IOException {
- if (retrievalCount.get() >= maxResults) {
- return Collections.emptySet();
- }
-
- LuceneUtil.sortDocsForRetrieval(docs);
-
- RecordReader reader = null;
- String lastStorageFilename = null;
- final Set<ProvenanceEventRecord> matchingRecords = new LinkedHashSet<>();
+ final AtomicInteger retrievalCount, final int maxResults, final int maxAttributeChars) throws IOException {
final long start = System.nanoTime();
- int logFileCount = 0;
-
- final Set<String> storageFilesToSkip = new HashSet<>();
- int eventsReadThisFile = 0;
- try {
- for (final Document d : docs) {
- final String storageFilename = d.getField(FieldNames.STORAGE_FILENAME).stringValue();
- if ( storageFilesToSkip.contains(storageFilename) ) {
- continue;
- }
-
- try {
- if (reader != null && storageFilename.equals(lastStorageFilename)) {
- matchingRecords.add(getRecord(d, reader));
- eventsReadThisFile++;
-
- if ( retrievalCount.incrementAndGet() >= maxResults ) {
- break;
- }
- } else {
- logger.debug("Opening log file {}", storageFilename);
-
- logFileCount++;
- if (reader != null) {
- reader.close();
- }
+ Set<ProvenanceEventRecord> matchingRecords = new LinkedHashSet<>();
+ if (retrievalCount.get() >= maxResults) {
+ return matchingRecords;
+ }
- final List<File> potentialFiles = LuceneUtil.getProvenanceLogFiles(storageFilename, allProvenanceLogFiles);
- if (potentialFiles.isEmpty()) {
- logger.warn("Could not find Provenance Log File with basename {} in the "
- + "Provenance Repository; assuming file has expired and continuing without it", storageFilename);
- storageFilesToSkip.add(storageFilename);
- continue;
- }
+ Map<String, List<Document>> byStorageNameDocGroups = LuceneUtil.groupDocsByStorageFileName(docs);
- if (potentialFiles.size() > 1) {
- throw new FileNotFoundException("Found multiple Provenance Log Files with basename " +
- storageFilename + " in the Provenance Repository");
- }
+ int eventsReadThisFile = 0;
+ int logFileCount = 0;
- for (final File file : potentialFiles) {
- try {
- if (reader != null) {
- logger.debug("Read {} records from previous file", eventsReadThisFile);
- }
-
- reader = RecordReaders.newRecordReader(file, allProvenanceLogFiles, maxAttributeChars);
- matchingRecords.add(getRecord(d, reader));
- eventsReadThisFile = 1;
-
- if ( retrievalCount.incrementAndGet() >= maxResults ) {
- break;
- }
- } catch (final IOException e) {
- throw new IOException("Failed to retrieve record " + d + " from Provenance File " + file + " due to " + e, e);
+ for (String storageFileName : byStorageNameDocGroups.keySet()) {
+ File provenanceEventFile = LuceneUtil.getProvenanceLogFile(storageFileName, allProvenanceLogFiles);
+ if (provenanceEventFile != null) {
+ try (RecordReader reader = RecordReaders.newRecordReader(provenanceEventFile, allProvenanceLogFiles,
+ maxAttributeChars)) {
+ for (Document document : byStorageNameDocGroups.get(storageFileName)) {
+ ProvenanceEventRecord eRec = this.getRecord(document, reader);
+ if (eRec != null) {
+ matchingRecords.add(eRec);
+ eventsReadThisFile++;
+
+ if (retrievalCount.incrementAndGet() >= maxResults) {
+ break;
}
}
}
- } finally {
- lastStorageFilename = storageFilename;
+ } catch (Exception e) {
+ logger.warn("Failed while trying to read Provenance Events. The event file '"
+ + provenanceEventFile.getAbsolutePath() +
+ "' may be missing or corrupted.", e);
}
- }
- } finally {
- if (reader != null) {
- reader.close();
+ } else {
+ logger.warn("Could not find Provenance Log File with "
+ + "basename {} in the Provenance Repository; assuming "
+ + "file has expired and continuing without it", storageFileName);
}
}
logger.debug("Read {} records from previous file", eventsReadThisFile);
final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
- logger.debug("Took {} ms to read {} events from {} prov log files", millis, matchingRecords.size(), logFileCount);
+ logger.debug("Took {} ms to read {} events from {} prov log files", millis, matchingRecords.size(),
+ logFileCount);
return matchingRecords;
}
-
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/a4d93c62/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/IndexSearch.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/IndexSearch.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/IndexSearch.java
index 7fcd8ab..b8661df 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/IndexSearch.java
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/IndexSearch.java
@@ -89,7 +89,7 @@ public class IndexSearch {
return sqr;
}
- final DocsReader docsReader = new DocsReader(repository.getConfiguration().getStorageDirectories());
+ final DocsReader docsReader = new DocsReader();
matchingRecords = docsReader.read(topDocs, searcher.getIndexReader(), repository.getAllLogFiles(), retrievedCount,
provenanceQuery.getMaxResults(), maxAttributeChars);
http://git-wip-us.apache.org/repos/asf/nifi/blob/a4d93c62/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/LineageQuery.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/LineageQuery.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/LineageQuery.java
index e9e6e63..ce60e03 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/LineageQuery.java
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/LineageQuery.java
@@ -93,7 +93,7 @@ public class LineageQuery {
final TopDocs uuidQueryTopDocs = searcher.search(query, MAX_QUERY_RESULTS);
final long searchEnd = System.nanoTime();
- final DocsReader docsReader = new DocsReader(repo.getConfiguration().getStorageDirectories());
+ final DocsReader docsReader = new DocsReader();
final Set<ProvenanceEventRecord> recs = docsReader.read(uuidQueryTopDocs, searcher.getIndexReader(), repo.getAllLogFiles(),
new AtomicInteger(0), Integer.MAX_VALUE, maxAttributeChars);
http://git-wip-us.apache.org/repos/asf/nifi/blob/a4d93c62/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/LuceneUtil.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/LuceneUtil.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/LuceneUtil.java
index c622ea1..aa50b94 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/LuceneUtil.java
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/LuceneUtil.java
@@ -22,7 +22,9 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import org.apache.nifi.processor.DataUnit;
import org.apache.nifi.provenance.SearchableFields;
@@ -160,4 +162,25 @@ public class LuceneUtil {
}
});
}
+
+ /**
+ * Will group documents based on the {@link FieldNames#STORAGE_FILENAME}.
+ *
+ * @param documents
+ * list of {@link Document}s
+ * @return a {@link Map} of document groups with
+ * {@link FieldNames#STORAGE_FILENAME} as key and {@link List} of
+ * {@link Document}s as value.
+ */
+ public static Map<String, List<Document>> groupDocsByStorageFileName(final List<Document> documents) {
+ Map<String, List<Document>> documentGroups = new HashMap<>();
+ for (Document document : documents) {
+ String fileName = document.get(FieldNames.STORAGE_FILENAME);
+ if (!documentGroups.containsKey(fileName)) {
+ documentGroups.put(fileName, new ArrayList<Document>());
+ }
+ documentGroups.get(fileName).add(document);
+ }
+ return documentGroups;
+ }
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/a4d93c62/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestPersistentProvenanceRepository.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestPersistentProvenanceRepository.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestPersistentProvenanceRepository.java
index 5e4aed0..02b9216 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestPersistentProvenanceRepository.java
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestPersistentProvenanceRepository.java
@@ -20,9 +20,11 @@ import static org.apache.nifi.provenance.TestUtil.createFlowFile;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
import java.io.File;
import java.io.FileFilter;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
@@ -36,6 +38,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.zip.GZIPOutputStream;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.core.SimpleAnalyzer;
@@ -48,6 +51,7 @@ import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.FSDirectory;
import org.apache.nifi.events.EventReporter;
+import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.provenance.lineage.EventNode;
import org.apache.nifi.provenance.lineage.Lineage;
import org.apache.nifi.provenance.lineage.LineageEdge;
@@ -869,6 +873,72 @@ public class TestPersistentProvenanceRepository {
}
}
+ /**
+ * Here the event file is simply corrupted by virtue of not having any event
+ * records while having correct headers
+ */
+ @Test
+ public void testWithWithEventFileMissingRecord() throws Exception {
+ File eventFile = this.prepCorruptedEventFileTests();
+
+ final Query query = new Query(UUID.randomUUID().toString());
+ query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "foo-*"));
+ query.setMaxResults(100);
+
+ DataOutputStream in = new DataOutputStream(new GZIPOutputStream(new FileOutputStream(eventFile)));
+ in.writeUTF("BlahBlah");
+ in.writeInt(4);
+ in.close();
+ assertTrue(eventFile.exists());
+ final QueryResult result = repo.queryEvents(query);
+ assertEquals(10, result.getMatchingEvents().size());
+ }
+
+ /**
+ * Here the event file is simply corrupted by virtue of being empty (0
+ * bytes)
+ */
+ @Test
+ public void testWithWithEventFileCorrupted() throws Exception {
+ File eventFile = this.prepCorruptedEventFileTests();
+
+ final Query query = new Query(UUID.randomUUID().toString());
+ query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "foo-*"));
+ query.setMaxResults(100);
+ DataOutputStream in = new DataOutputStream(new GZIPOutputStream(new FileOutputStream(eventFile)));
+ in.close();
+ final QueryResult result = repo.queryEvents(query);
+ assertEquals(10, result.getMatchingEvents().size());
+ }
+
+ private File prepCorruptedEventFileTests() throws Exception {
+ RepositoryConfiguration config = createConfiguration();
+ config.setMaxStorageCapacity(1024L * 1024L);
+ config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
+ config.setMaxEventFileCapacity(1024L * 1024L);
+ config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
+ config.setDesiredIndexSize(10);
+
+ repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
+ repo.initialize(getEventReporter());
+
+ String uuid = UUID.randomUUID().toString();
+ for (int i = 0; i < 20; i++) {
+ ProvenanceEventRecord record = repo.eventBuilder().fromFlowFile(mock(FlowFile.class))
+ .setEventType(ProvenanceEventType.CREATE).setComponentId("foo-" + i).setComponentType("myComponent")
+ .setFlowFileUUID(uuid).build();
+ repo.registerEvent(record);
+ if (i == 9) {
+ repo.waitForRollover();
+ Thread.sleep(2000L);
+ }
+ }
+ repo.waitForRollover();
+ File eventFile = new File(config.getStorageDirectories().get(0), "10.prov.gz");
+ assertTrue(eventFile.delete());
+ return eventFile;
+ }
+
@Test
public void testIndexDirectoryRemoved() throws InterruptedException, IOException, ParseException {
final RepositoryConfiguration config = createConfiguration();
[48/50] [abbrv] nifi git commit: NIFI-655: - Refactoring web security
to use Spring Security Java Configuration. - Introducing security in Web UI
in order to get JWT.
Posted by mc...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/KeyDAO.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/KeyDAO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/KeyDAO.java
new file mode 100644
index 0000000..2a24e0b
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/KeyDAO.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.dao;
+
+import org.apache.nifi.key.Key;
+
+/**
+ * Key data access.
+ */
+public interface KeyDAO {
+
+ /**
+ * Gets the key for the specified user identity. Returns null if no key exists for the key id.
+ *
+ * @param id The key id
+ * @return The key or null
+ */
+ Key findKeyById(int id);
+
+ /**
+ * Gets the latest key for the specified identity. Returns null if no key exists for the user identity.
+ *
+ * @param identity The identity
+ * @return The key or null
+ */
+ Key findLatestKeyByIdentity(String identity);
+
+ /**
+ * Creates a key for the specified user identity.
+ *
+ * @param identity The user identity
+ * @return The key
+ */
+ Key createKey(String identity);
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/impl/DAOFactoryImpl.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/impl/DAOFactoryImpl.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/impl/DAOFactoryImpl.java
index 2f3de0e..940e364 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/impl/DAOFactoryImpl.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/impl/DAOFactoryImpl.java
@@ -20,6 +20,7 @@ import java.sql.Connection;
import org.apache.nifi.admin.dao.ActionDAO;
import org.apache.nifi.admin.dao.AuthorityDAO;
import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.admin.dao.KeyDAO;
import org.apache.nifi.admin.dao.UserDAO;
/**
@@ -48,4 +49,9 @@ public class DAOFactoryImpl implements DAOFactory {
return new StandardUserDAO(connection);
}
+ @Override
+ public KeyDAO getKeyDAO() {
+ return new StandardKeyDAO(connection);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/impl/StandardActionDAO.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/impl/StandardActionDAO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/impl/StandardActionDAO.java
index fe693e1..8fdfd34 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/impl/StandardActionDAO.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/impl/StandardActionDAO.java
@@ -63,7 +63,7 @@ public class StandardActionDAO implements ActionDAO {
// action table
// ------------
private static final String INSERT_ACTION = "INSERT INTO ACTION ("
- + "USER_DN, USER_NAME, SOURCE_ID, SOURCE_NAME, SOURCE_TYPE, OPERATION, ACTION_TIMESTAMP"
+ + "IDENTITY, USER_NAME, SOURCE_ID, SOURCE_NAME, SOURCE_TYPE, OPERATION, ACTION_TIMESTAMP"
+ ") VALUES ("
+ "?, "
+ "?, "
@@ -216,8 +216,8 @@ public class StandardActionDAO implements ActionDAO {
try {
// obtain a statement to insert to the action table
statement = connection.prepareStatement(INSERT_ACTION, Statement.RETURN_GENERATED_KEYS);
- statement.setString(1, StringUtils.left(action.getUserIdentity(), 255));
- statement.setString(2, StringUtils.left(action.getUserName(), 100));
+ statement.setString(1, StringUtils.left(action.getUserIdentity(), 4096));
+ statement.setString(2, StringUtils.left(action.getUserName(), 4096));
statement.setString(3, action.getSourceId());
statement.setString(4, StringUtils.left(action.getSourceName(), 1000));
statement.setString(5, action.getSourceType().toString());
@@ -561,7 +561,7 @@ public class StandardActionDAO implements ActionDAO {
FlowChangeAction action = new FlowChangeAction();
action.setId(actionId);
- action.setUserIdentity(rs.getString("USER_DN"));
+ action.setUserIdentity(rs.getString("IDENTITY"));
action.setUserName(rs.getString("USER_NAME"));
action.setOperation(Operation.valueOf(rs.getString("OPERATION")));
action.setTimestamp(new Date(rs.getTimestamp("ACTION_TIMESTAMP").getTime()));
@@ -635,7 +635,7 @@ public class StandardActionDAO implements ActionDAO {
// populate the action
action = new FlowChangeAction();
action.setId(rs.getInt("ID"));
- action.setUserIdentity(rs.getString("USER_DN"));
+ action.setUserIdentity(rs.getString("IDENTITY"));
action.setUserName(rs.getString("USER_NAME"));
action.setOperation(operation);
action.setTimestamp(new Date(rs.getTimestamp("ACTION_TIMESTAMP").getTime()));
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/impl/StandardKeyDAO.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/impl/StandardKeyDAO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/impl/StandardKeyDAO.java
new file mode 100644
index 0000000..f4bdc1d
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/impl/StandardKeyDAO.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.dao.impl;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.UUID;
+import org.apache.nifi.admin.RepositoryUtils;
+import org.apache.nifi.admin.dao.DataAccessException;
+import org.apache.nifi.admin.dao.KeyDAO;
+import org.apache.nifi.key.Key;
+
+/**
+ *
+ */
+public class StandardKeyDAO implements KeyDAO {
+
+ private static final String SELECT_KEY_FOR_USER_BY_ID = "SELECT ID, IDENTITY, KEY "
+ + "FROM KEY "
+ + "WHERE ID = ?";
+
+ private static final String SELECT_KEY_FOR_USER_BY_IDENTITY = "SELECT ID, IDENTITY, KEY "
+ + "FROM KEY "
+ + "WHERE IDENTITY = ?";
+
+ private static final String INSERT_KEY = "INSERT INTO KEY ("
+ + "IDENTITY, KEY"
+ + ") VALUES ("
+ + "?, ?"
+ + ")";
+
+ private final Connection connection;
+
+ public StandardKeyDAO(Connection connection) {
+ this.connection = connection;
+ }
+
+ @Override
+ public Key findKeyById(int id) {
+ Key key = null;
+
+ PreparedStatement statement = null;
+ ResultSet rs = null;
+ try {
+ // add each authority for the specified user
+ statement = connection.prepareStatement(SELECT_KEY_FOR_USER_BY_ID);
+ statement.setInt(1, id);
+
+ // execute the query
+ rs = statement.executeQuery();
+
+ // if the key was found, add it
+ if (rs.next()) {
+ key = new Key();
+ key.setId(rs.getInt("ID"));
+ key.setIdentity(rs.getString("IDENTITY"));
+ key.setKey(rs.getString("KEY"));
+ }
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } finally {
+ RepositoryUtils.closeQuietly(rs);
+ RepositoryUtils.closeQuietly(statement);
+ }
+
+ return key;
+ }
+
+ @Override
+ public Key findLatestKeyByIdentity(String identity) {
+ if (identity == null) {
+ throw new IllegalArgumentException("Specified identity cannot be null.");
+ }
+
+ Key key = null;
+
+ PreparedStatement statement = null;
+ ResultSet rs = null;
+ try {
+ // add each authority for the specified user
+ statement = connection.prepareStatement(SELECT_KEY_FOR_USER_BY_IDENTITY);
+ statement.setString(1, identity);
+
+ // execute the query
+ rs = statement.executeQuery();
+
+ // if the key was found, add it
+ if (rs.next()) {
+ key = new Key();
+ key.setId(rs.getInt("ID"));
+ key.setIdentity(rs.getString("IDENTITY"));
+ key.setKey(rs.getString("KEY"));
+ }
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } finally {
+ RepositoryUtils.closeQuietly(rs);
+ RepositoryUtils.closeQuietly(statement);
+ }
+
+ return key;
+ }
+
+ @Override
+ public Key createKey(final String identity) {
+ PreparedStatement statement = null;
+ ResultSet rs = null;
+ try {
+ final String keyValue = UUID.randomUUID().toString();
+
+ // add each authority for the specified user
+ statement = connection.prepareStatement(INSERT_KEY, Statement.RETURN_GENERATED_KEYS);
+ statement.setString(1, identity);
+ statement.setString(2, keyValue);
+
+ // insert the key
+ int updateCount = statement.executeUpdate();
+ rs = statement.getGeneratedKeys();
+
+ // verify the results
+ if (updateCount == 1 && rs.next()) {
+ final Key key = new Key();
+ key.setId(rs.getInt(1));
+ key.setIdentity(identity);
+ key.setKey(keyValue);
+ return key;
+ } else {
+ throw new DataAccessException("Unable to add key for user.");
+ }
+ } catch (SQLException sqle) {
+ throw new DataAccessException(sqle);
+ } finally {
+ RepositoryUtils.closeQuietly(rs);
+ RepositoryUtils.closeQuietly(statement);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/impl/StandardUserDAO.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/impl/StandardUserDAO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/impl/StandardUserDAO.java
index f6a62df..20356e3 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/impl/StandardUserDAO.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/impl/StandardUserDAO.java
@@ -45,9 +45,9 @@ public class StandardUserDAO implements UserDAO {
+ "FROM USER U "
+ "WHERE U.STATUS = 'PENDING'";
- private static final String SELECT_USER_BY_DN = "SELECT "
+ private static final String SELECT_USER_BY_USER = "SELECT "
+ "U.ID, "
- + "U.DN, "
+ + "U.IDENTITY, "
+ "U.USER_NAME, "
+ "U.USER_GROUP, "
+ "U.CREATION, "
@@ -59,11 +59,11 @@ public class StandardUserDAO implements UserDAO {
+ "FROM USER U "
+ "LEFT JOIN AUTHORITY A " // ensures that users without authorities are still matched
+ "ON U.ID = A.USER_ID "
- + "WHERE U.DN = ?";
+ + "WHERE U.IDENTITY = ?";
private static final String SELECT_USER_BY_ID = "SELECT "
+ "U.ID, "
- + "U.DN, "
+ + "U.IDENTITY, "
+ "U.USER_NAME, "
+ "U.USER_GROUP, "
+ "U.CREATION, "
@@ -79,7 +79,7 @@ public class StandardUserDAO implements UserDAO {
private static final String SELECT_USERS = "SELECT "
+ "U.ID, "
- + "U.DN, "
+ + "U.IDENTITY, "
+ "U.USER_NAME, "
+ "U.USER_GROUP, "
+ "U.CREATION, "
@@ -91,7 +91,7 @@ public class StandardUserDAO implements UserDAO {
+ "FROM USER U "
+ "LEFT JOIN AUTHORITY A " // ensures that users without authorities are still matched
+ "ON U.ID = A.USER_ID "
- + "WHERE U.DN <> ?";
+ + "WHERE U.IDENTITY <> ?";
private static final String SELECT_USER_GROUPS = "SELECT DISTINCT "
+ "U.USER_GROUP "
@@ -99,7 +99,7 @@ public class StandardUserDAO implements UserDAO {
private static final String SELECT_USER_GROUP = "SELECT "
+ "U.ID, "
- + "U.DN, "
+ + "U.IDENTITY, "
+ "U.USER_NAME, "
+ "U.USER_GROUP, "
+ "U.CREATION, "
@@ -111,10 +111,10 @@ public class StandardUserDAO implements UserDAO {
+ "FROM USER U "
+ "LEFT JOIN AUTHORITY A " // ensures that users without authorities are still matched
+ "ON U.ID = A.USER_ID "
- + "WHERE U.DN <> ? AND U.USER_GROUP = ?";
+ + "WHERE U.IDENTITY <> ? AND U.USER_GROUP = ?";
private static final String INSERT_USER = "INSERT INTO USER ("
- + "ID, DN, USER_NAME, USER_GROUP, CREATION, LAST_VERIFIED, JUSTIFICATION, STATUS"
+ + "ID, IDENTITY, USER_NAME, USER_GROUP, CREATION, LAST_VERIFIED, JUSTIFICATION, STATUS"
+ ") VALUES ("
+ "?, "
+ "?, "
@@ -127,7 +127,7 @@ public class StandardUserDAO implements UserDAO {
+ ")";
private static final String UPDATE_USER = "UPDATE USER SET "
- + "DN = ?, "
+ + "IDENTITY = ?, "
+ "USER_NAME = ?, "
+ "USER_GROUP = ?, "
+ "LAST_ACCESSED = ?, "
@@ -194,7 +194,7 @@ public class StandardUserDAO implements UserDAO {
try {
// create the connection and obtain a statement
statement = connection.prepareStatement(SELECT_USERS);
- statement.setString(1, NiFiUser.ANONYMOUS_USER_DN);
+ statement.setString(1, NiFiUser.ANONYMOUS_USER_IDENTITY);
// execute the query
rs = statement.executeQuery();
@@ -211,7 +211,7 @@ public class StandardUserDAO implements UserDAO {
if (user == null || !userId.equals(user.getId())) {
user = new NiFiUser();
user.setId(userId);
- user.setDn(rs.getString("DN"));
+ user.setIdentity(rs.getString("IDENTITY"));
user.setUserName(rs.getString("USER_NAME"));
user.setUserGroup(rs.getString("USER_GROUP"));
user.setJustification(rs.getString("JUSTIFICATION"));
@@ -287,7 +287,7 @@ public class StandardUserDAO implements UserDAO {
try {
// create the connection and obtain a statement
statement = connection.prepareStatement(SELECT_USER_GROUP);
- statement.setString(1, NiFiUser.ANONYMOUS_USER_DN);
+ statement.setString(1, NiFiUser.ANONYMOUS_USER_IDENTITY);
statement.setString(2, group);
// execute the query
@@ -305,7 +305,7 @@ public class StandardUserDAO implements UserDAO {
if (user == null || !userId.equals(user.getId())) {
user = new NiFiUser();
user.setId(userId);
- user.setDn(rs.getString("DN"));
+ user.setIdentity(rs.getString("IDENTITY"));
user.setUserName(rs.getString("USER_NAME"));
user.setUserGroup(rs.getString("USER_GROUP"));
user.setJustification(rs.getString("JUSTIFICATION"));
@@ -366,7 +366,7 @@ public class StandardUserDAO implements UserDAO {
if (user == null) {
user = new NiFiUser();
user.setId(rs.getString("ID"));
- user.setDn(rs.getString("DN"));
+ user.setIdentity(rs.getString("IDENTITY"));
user.setUserName(rs.getString("USER_NAME"));
user.setUserGroup(rs.getString("USER_GROUP"));
user.setJustification(rs.getString("JUSTIFICATION"));
@@ -409,7 +409,7 @@ public class StandardUserDAO implements UserDAO {
ResultSet rs = null;
try {
// create the connection and obtain a statement
- statement = connection.prepareStatement(SELECT_USER_BY_DN);
+ statement = connection.prepareStatement(SELECT_USER_BY_USER);
statement.setString(1, dn);
// execute the query
@@ -424,7 +424,7 @@ public class StandardUserDAO implements UserDAO {
if (user == null) {
user = new NiFiUser();
user.setId(rs.getString("ID"));
- user.setDn(rs.getString("DN"));
+ user.setIdentity(rs.getString("IDENTITY"));
user.setUserName(rs.getString("USER_NAME"));
user.setUserGroup(rs.getString("USER_GROUP"));
user.setJustification(rs.getString("JUSTIFICATION"));
@@ -463,20 +463,25 @@ public class StandardUserDAO implements UserDAO {
@Override
public NiFiUser createUser(NiFiUser user) throws DataAccessException {
- if (user.getDn() == null) {
- throw new IllegalArgumentException("User dn must be specified.");
+ if (user.getIdentity() == null) {
+ throw new IllegalArgumentException("User identity must be specified.");
+ }
+
+ // ensure the user identity is not too lengthy
+ if (user.getIdentity().length() > 4096) {
+ throw new IllegalArgumentException("User identity must be less than 4096 characters.");
}
PreparedStatement statement = null;
ResultSet rs = null;
try {
- final String id = UUID.nameUUIDFromBytes(user.getDn().getBytes(StandardCharsets.UTF_8)).toString();
+ final String id = UUID.nameUUIDFromBytes(user.getIdentity().getBytes(StandardCharsets.UTF_8)).toString();
// create a statement
statement = connection.prepareStatement(INSERT_USER, Statement.RETURN_GENERATED_KEYS);
statement.setString(1, id);
- statement.setString(2, StringUtils.left(user.getDn(), 255));
- statement.setString(3, StringUtils.left(user.getUserName(), 100));
+ statement.setString(2, StringUtils.left(user.getIdentity(), 4096));
+ statement.setString(3, StringUtils.left(user.getUserName(), 4096));
statement.setString(4, StringUtils.left(user.getUserGroup(), 100));
if (user.getLastVerified() != null) {
statement.setTimestamp(5, new java.sql.Timestamp(user.getLastVerified().getTime()));
@@ -531,8 +536,8 @@ public class StandardUserDAO implements UserDAO {
try {
// create a statement
statement = connection.prepareStatement(UPDATE_USER);
- statement.setString(1, StringUtils.left(user.getDn(), 255));
- statement.setString(2, StringUtils.left(user.getUserName(), 100));
+ statement.setString(1, StringUtils.left(user.getIdentity(), 4096));
+ statement.setString(2, StringUtils.left(user.getUserName(), 4096));
statement.setString(3, StringUtils.left(user.getUserGroup(), 100));
statement.setString(6, StringUtils.left(user.getJustification(), 500));
statement.setString(7, user.getStatus().toString());
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/KeyService.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/KeyService.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/KeyService.java
new file mode 100644
index 0000000..ae64c41
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/KeyService.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service;
+
+import org.apache.nifi.key.Key;
+
+/**
+ * Supports retrieving and issues keys for signing user tokens.
+ */
+public interface KeyService {
+
+ /**
+ * Gets a key for the specified user identity. Returns null if the user has not had a key issued
+ *
+ * @param id The key id
+ * @return The key or null
+ */
+ Key getKey(int id);
+
+ /**
+ * Gets a key for the specified user identity. If a key does not exist, one will be created.
+ *
+ * @param identity The user identity
+ * @return The key
+ * @throws AdministrationException if it failed to get/create the key
+ */
+ Key getOrCreateKey(String identity);
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/AbstractUserAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/AbstractUserAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/AbstractUserAction.java
index b970dc1..69c6c1f 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/AbstractUserAction.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/AbstractUserAction.java
@@ -76,14 +76,14 @@ public abstract class AbstractUserAction<T> implements AdministrationAction<T> {
*/
protected void verifyAccount(AuthorityProvider authorityProvider, NiFiUser user) {
// load the roles for the user
- Set<Authority> authorities = authorityProvider.getAuthorities(user.getDn());
+ Set<Authority> authorities = authorityProvider.getAuthorities(user.getIdentity());
// update the user's authorities
user.getAuthorities().clear();
user.getAuthorities().addAll(authorities);
// get the user group
- user.setUserGroup(authorityProvider.getGroupForUser(user.getDn()));
+ user.setUserGroup(authorityProvider.getGroupForUser(user.getIdentity()));
// update the users status in case they were previously pending or disabled
user.setStatus(AccountStatus.ACTIVE);
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/AuthorizeUserAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/AuthorizeUserAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/AuthorizeUserAction.java
index fe32772..ed4dfa1 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/AuthorizeUserAction.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/AuthorizeUserAction.java
@@ -37,11 +37,11 @@ import org.apache.nifi.user.NiFiUser;
*/
public class AuthorizeUserAction extends AbstractUserAction<NiFiUser> {
- private final String dn;
+ private final String identity;
private final int cacheDurationSeconds;
- public AuthorizeUserAction(String dn, int cacheDurationSeconds) {
- this.dn = dn;
+ public AuthorizeUserAction(String identity, int cacheDurationSeconds) {
+ this.identity = identity;
this.cacheDurationSeconds = cacheDurationSeconds;
}
@@ -50,14 +50,14 @@ public class AuthorizeUserAction extends AbstractUserAction<NiFiUser> {
UserDAO userDao = daoFactory.getUserDAO();
// get the user
- NiFiUser user = userDao.findUserByDn(dn);
+ NiFiUser user = userDao.findUserByDn(identity);
// verify the user was found
if (user == null) {
// determine whether this users exists
boolean doesDnExist = false;
try {
- doesDnExist = authorityProvider.doesDnExist(dn);
+ doesDnExist = authorityProvider.doesDnExist(identity);
} catch (AuthorityAccessException aae) {
throw new AdministrationException(String.format("Unable to access authority details: %s", aae.getMessage()), aae);
}
@@ -66,8 +66,8 @@ public class AuthorizeUserAction extends AbstractUserAction<NiFiUser> {
if (doesDnExist) {
// create the user
user = new NiFiUser();
- user.setDn(dn);
- user.setUserName(CertificateUtils.extractUsername(dn));
+ user.setIdentity(identity);
+ user.setUserName(CertificateUtils.extractUsername(identity));
user.setJustification("User details specified by authority provider.");
try {
@@ -86,12 +86,12 @@ public class AuthorizeUserAction extends AbstractUserAction<NiFiUser> {
createUser.execute(daoFactory, authorityProvider);
} catch (UnknownIdentityException uie) {
// strange since the provider just reported this dn existed but handleing anyways...
- throw new AccountNotFoundException(String.format("Unable to verify access for %s.", dn));
+ throw new AccountNotFoundException(String.format("Unable to verify access for %s.", identity));
} catch (AuthorityAccessException aae) {
throw new AdministrationException(String.format("Unable to access authority details: %s", aae.getMessage()), aae);
}
} else {
- throw new AccountNotFoundException(String.format("Unable to verify access for %s.", dn));
+ throw new AccountNotFoundException(String.format("Unable to verify access for %s.", identity));
}
} else {
Throwable providerError = null;
@@ -134,7 +134,7 @@ public class AuthorizeUserAction extends AbstractUserAction<NiFiUser> {
updateUserAuthorities.execute(daoFactory, authorityProvider);
if (providerError != null) {
- throw new AccountDisabledException(String.format("User credentials for %s were not found. This account has been disabled.", user.getDn()), providerError);
+ throw new AccountDisabledException(String.format("User credentials for %s were not found. This account has been disabled.", user.getIdentity()), providerError);
}
}
@@ -165,9 +165,9 @@ public class AuthorizeUserAction extends AbstractUserAction<NiFiUser> {
*/
private void checkAccountStatus(NiFiUser user) {
if (AccountStatus.DISABLED.equals(user.getStatus())) {
- throw new AccountDisabledException(String.format("Account for %s is disabled.", user.getDn()));
+ throw new AccountDisabledException(String.format("The account for %s has been disabled.", user.getIdentity()));
} else if (AccountStatus.PENDING.equals(user.getStatus())) {
- throw new AccountPendingException(String.format("Account for %s is pending.", user.getDn()));
+ throw new AccountPendingException(String.format("The account for %s is currently pending approval.", user.getIdentity()));
}
}
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/DisableUserAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/DisableUserAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/DisableUserAction.java
index c31f107..9e9b798 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/DisableUserAction.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/DisableUserAction.java
@@ -63,12 +63,12 @@ public class DisableUserAction implements AdministrationAction<NiFiUser> {
try {
// revoke the user in the authority provider
- authorityProvider.revokeUser(user.getDn());
+ authorityProvider.revokeUser(user.getIdentity());
} catch (UnknownIdentityException uie) {
// user identity is not known
- logger.info(String.format("User %s has already been removed from the authority provider.", user.getDn()));
+ logger.info(String.format("User %s has already been removed from the authority provider.", user.getIdentity()));
} catch (AuthorityAccessException aae) {
- throw new AdministrationException(String.format("Unable to revoke user '%s': %s", user.getDn(), aae.getMessage()), aae);
+ throw new AdministrationException(String.format("Unable to revoke user '%s': %s", user.getIdentity(), aae.getMessage()), aae);
}
return user;
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/GetKeyByIdAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/GetKeyByIdAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/GetKeyByIdAction.java
new file mode 100644
index 0000000..8763b9d
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/GetKeyByIdAction.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.authorization.AuthorityProvider;
+
+import org.apache.nifi.admin.dao.KeyDAO;
+import org.apache.nifi.key.Key;
+
+/**
+ * Gets a key for the specified key id.
+ */
+public class GetKeyByIdAction implements AdministrationAction<Key> {
+
+ private final int id;
+
+ public GetKeyByIdAction(int id) {
+ this.id = id;
+ }
+
+ @Override
+ public Key execute(DAOFactory daoFactory, AuthorityProvider authorityProvider) {
+ final KeyDAO keyDao = daoFactory.getKeyDAO();
+ return keyDao.findKeyById(id);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/GetKeyByIdentityAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/GetKeyByIdentityAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/GetKeyByIdentityAction.java
new file mode 100644
index 0000000..9bcb0b3
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/GetKeyByIdentityAction.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.authorization.AuthorityProvider;
+
+import org.apache.nifi.admin.dao.KeyDAO;
+import org.apache.nifi.key.Key;
+
+/**
+ * Gets a key for the specified key id.
+ */
+public class GetKeyByIdentityAction implements AdministrationAction<Key> {
+
+ private final String identity;
+
+ public GetKeyByIdentityAction(String identity) {
+ this.identity = identity;
+ }
+
+ @Override
+ public Key execute(DAOFactory daoFactory, AuthorityProvider authorityProvider) {
+ final KeyDAO keyDao = daoFactory.getKeyDAO();
+ return keyDao.findLatestKeyByIdentity(identity);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/GetOrCreateKeyAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/GetOrCreateKeyAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/GetOrCreateKeyAction.java
new file mode 100644
index 0000000..bb85b6f
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/GetOrCreateKeyAction.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.action;
+
+import org.apache.nifi.admin.dao.DAOFactory;
+import org.apache.nifi.authorization.AuthorityProvider;
+
+import org.apache.nifi.admin.dao.KeyDAO;
+import org.apache.nifi.key.Key;
+
+/**
+ * Gets a key for the specified user identity.
+ */
+public class GetOrCreateKeyAction implements AdministrationAction<Key> {
+
+ private final String identity;
+
+ public GetOrCreateKeyAction(String identity) {
+ this.identity = identity;
+ }
+
+ @Override
+ public Key execute(DAOFactory daoFactory, AuthorityProvider authorityProvider) {
+ final KeyDAO keyDao = daoFactory.getKeyDAO();
+
+ Key key = keyDao.findLatestKeyByIdentity(identity);
+ if (key == null) {
+ key = keyDao.createKey(identity);
+ }
+
+ return key;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/RequestUserAccountAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/RequestUserAccountAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/RequestUserAccountAction.java
index 3dce6d9..198a32d 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/RequestUserAccountAction.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/RequestUserAccountAction.java
@@ -30,11 +30,11 @@ import org.apache.nifi.user.NiFiUser;
*/
public class RequestUserAccountAction implements AdministrationAction<NiFiUser> {
- private final String dn;
+ private final String identity;
private final String justification;
- public RequestUserAccountAction(String dn, String justification) {
- this.dn = dn;
+ public RequestUserAccountAction(String identity, String justification) {
+ this.identity = identity;
this.justification = justification;
}
@@ -43,15 +43,15 @@ public class RequestUserAccountAction implements AdministrationAction<NiFiUser>
UserDAO userDao = daoFactory.getUserDAO();
// determine if this user already exists
- NiFiUser user = userDao.findUserByDn(dn);
+ NiFiUser user = userDao.findUserByDn(identity);
if (user != null) {
- throw new IllegalArgumentException(String.format("User account for %s already exists.", dn));
+ throw new IllegalArgumentException(String.format("User account for %s already exists.", identity));
}
// create the user
user = new NiFiUser();
- user.setDn(dn);
- user.setUserName(CertificateUtils.extractUsername(dn));
+ user.setIdentity(identity);
+ user.setUserName(CertificateUtils.extractUsername(identity));
user.setJustification(justification);
user.setStatus(AccountStatus.PENDING);
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/SeedUserAccountsAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/SeedUserAccountsAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/SeedUserAccountsAction.java
index 6665ac0..c16cc71 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/SeedUserAccountsAction.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/SeedUserAccountsAction.java
@@ -46,7 +46,7 @@ public class SeedUserAccountsAction extends AbstractUserAction<Void> {
@Override
public Void execute(DAOFactory daoFactory, AuthorityProvider authorityProvider) throws DataAccessException {
UserDAO userDao = daoFactory.getUserDAO();
- Set<String> authorizedDns = new HashSet<>();
+ Set<String> authorizedIdentities = new HashSet<>();
// get the current user cache
final Set<NiFiUser> existingUsers;
@@ -62,7 +62,7 @@ public class SeedUserAccountsAction extends AbstractUserAction<Void> {
try {
// all users for all roles
for (final Authority authority : Authority.values()) {
- authorizedDns.addAll(authorityProvider.getUsers(authority));
+ authorizedIdentities.addAll(authorityProvider.getUsers(authority));
}
} catch (AuthorityAccessException aae) {
// unable to access the authority provider... honor the cache
@@ -73,25 +73,25 @@ public class SeedUserAccountsAction extends AbstractUserAction<Void> {
final Set<NiFiUser> accountsToRevoke = new HashSet<>(existingUsers);
// persist the users
- for (String dn : authorizedDns) {
+ for (String identity : authorizedIdentities) {
NiFiUser user = null;
try {
// locate the user for this dn
- user = userDao.findUserByDn(dn);
+ user = userDao.findUserByDn(identity);
boolean newAccount = false;
// if the user does not exist, create a new account
if (user == null) {
- logger.info(String.format("Creating user account: %s", dn));
+ logger.info(String.format("Creating user account: %s", identity));
newAccount = true;
// create the user
user = new NiFiUser();
- user.setDn(dn);
- user.setUserName(CertificateUtils.extractUsername(dn));
+ user.setIdentity(identity);
+ user.setUserName(CertificateUtils.extractUsername(identity));
user.setJustification("User details specified by authority provider.");
} else {
- logger.info(String.format("User account already created: %s. Updating authorities...", dn));
+ logger.info(String.format("User account already created: %s. Updating authorities...", identity));
}
// verify the account
@@ -142,7 +142,7 @@ public class SeedUserAccountsAction extends AbstractUserAction<Void> {
}
try {
- logger.info(String.format("User not authorized with configured provider: %s. Disabling account...", user.getDn()));
+ logger.info(String.format("User not authorized with configured provider: %s. Disabling account...", user.getIdentity()));
// disable the account and reset its last verified timestamp since it was not found
// in the current configured authority provider
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UngroupUserAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UngroupUserAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UngroupUserAction.java
index 01eaf5f..2604a47 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UngroupUserAction.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UngroupUserAction.java
@@ -56,11 +56,11 @@ public class UngroupUserAction extends AbstractUserAction<Void> {
try {
// update the authority provider
- authorityProvider.ungroupUser(user.getDn());
+ authorityProvider.ungroupUser(user.getIdentity());
} catch (UnknownIdentityException uie) {
- throw new AccountNotFoundException(String.format("Unable to ungroup user '%s': %s", user.getDn(), uie.getMessage()), uie);
+ throw new AccountNotFoundException(String.format("Unable to ungroup user '%s': %s", user.getIdentity(), uie.getMessage()), uie);
} catch (AuthorityAccessException aae) {
- throw new AdministrationException(String.format("Unable to ungroup user '%s': %s", user.getDn(), aae.getMessage()), aae);
+ throw new AdministrationException(String.format("Unable to ungroup user '%s': %s", user.getIdentity(), aae.getMessage()), aae);
}
return null;
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UpdateUserAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UpdateUserAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UpdateUserAction.java
index cef21d7..ecb91e6 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UpdateUserAction.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UpdateUserAction.java
@@ -61,41 +61,41 @@ public class UpdateUserAction extends AbstractUserAction<NiFiUser> {
}
// determine whether this users exists
- boolean doesDnExist = false;
+ boolean doesIdentityExist = false;
try {
- doesDnExist = authorityProvider.doesDnExist(user.getDn());
+ doesIdentityExist = authorityProvider.doesDnExist(user.getIdentity());
} catch (AuthorityAccessException aae) {
throw new AdministrationException(String.format("Unable to access authority details: %s", aae.getMessage()), aae);
}
// if the user already doesn't exist, add them
- if (!doesDnExist) {
+ if (!doesIdentityExist) {
try {
// add the account account and group if necessary
- authorityProvider.addUser(user.getDn(), user.getUserGroup());
+ authorityProvider.addUser(user.getIdentity(), user.getUserGroup());
} catch (final IdentityAlreadyExistsException iaee) {
- logger.warn(String.format("User '%s' already exists in the authority provider. Continuing with user update.", user.getDn()));
+ logger.warn(String.format("User '%s' already exists in the authority provider. Continuing with user update.", user.getIdentity()));
} catch (AuthorityAccessException aae) {
- throw new AdministrationException(String.format("Unable to access authorities for '%s': %s", user.getDn(), aae.getMessage()), aae);
+ throw new AdministrationException(String.format("Unable to access authorities for '%s': %s", user.getIdentity(), aae.getMessage()), aae);
}
}
try {
// update the authority provider as approprivate
- authorityProvider.setAuthorities(user.getDn(), authorities);
+ authorityProvider.setAuthorities(user.getIdentity(), authorities);
} catch (UnknownIdentityException uie) {
- throw new AccountNotFoundException(String.format("Unable to modify authorities for '%s': %s.", user.getDn(), uie.getMessage()), uie);
+ throw new AccountNotFoundException(String.format("Unable to modify authorities for '%s': %s.", user.getIdentity(), uie.getMessage()), uie);
} catch (AuthorityAccessException aae) {
- throw new AdministrationException(String.format("Unable to access authorities for '%s': %s.", user.getDn(), aae.getMessage()), aae);
+ throw new AdministrationException(String.format("Unable to access authorities for '%s': %s.", user.getIdentity(), aae.getMessage()), aae);
}
try {
// get the user group
- user.setUserGroup(authorityProvider.getGroupForUser(user.getDn()));
+ user.setUserGroup(authorityProvider.getGroupForUser(user.getIdentity()));
} catch (UnknownIdentityException uie) {
- throw new AccountNotFoundException(String.format("Unable to determine the group for '%s': %s.", user.getDn(), uie.getMessage()), uie);
+ throw new AccountNotFoundException(String.format("Unable to determine the group for '%s': %s.", user.getIdentity(), uie.getMessage()), uie);
} catch (AuthorityAccessException aae) {
- throw new AdministrationException(String.format("Unable to access the group for '%s': %s.", user.getDn(), aae.getMessage()), aae);
+ throw new AdministrationException(String.format("Unable to access the group for '%s': %s.", user.getIdentity(), aae.getMessage()), aae);
}
// since all the authorities were updated accordingly, set the authorities
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UpdateUserGroupAction.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UpdateUserGroupAction.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UpdateUserGroupAction.java
index 56b214c..1d7941f 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UpdateUserGroupAction.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/action/UpdateUserGroupAction.java
@@ -61,7 +61,7 @@ public class UpdateUserGroupAction extends AbstractUserAction<Void> {
// record the new users being added to this group
final Set<NiFiUser> newUsers = new HashSet<>();
- final Set<String> newUserDns = new HashSet<>();
+ final Set<String> newUserIdentities = new HashSet<>();
// if the user ids have been specified we need to create/update a group using the specified group name
if (userIds != null) {
@@ -81,13 +81,13 @@ public class UpdateUserGroupAction extends AbstractUserAction<Void> {
try {
// if the user is unknown to the authority provider we cannot continue
- if (!authorityProvider.doesDnExist(user.getDn()) || AccountStatus.DISABLED.equals(user.getStatus())) {
- throw new IllegalStateException(String.format("Unable to group these users because access for '%s' is not %s.", user.getDn(), AccountStatus.ACTIVE.toString()));
+ if (!authorityProvider.doesDnExist(user.getIdentity()) || AccountStatus.DISABLED.equals(user.getStatus())) {
+ throw new IllegalStateException(String.format("Unable to group these users because access for '%s' is not %s.", user.getIdentity(), AccountStatus.ACTIVE.toString()));
}
// record the user being added to this group
newUsers.add(user);
- newUserDns.add(user.getDn());
+ newUserIdentities.add(user.getIdentity());
} catch (final AuthorityAccessException aae) {
throw new AdministrationException(String.format("Unable to access authority details: %s", aae.getMessage()), aae);
}
@@ -95,11 +95,11 @@ public class UpdateUserGroupAction extends AbstractUserAction<Void> {
try {
// update the authority provider
- authorityProvider.setUsersGroup(newUserDns, group);
+ authorityProvider.setUsersGroup(newUserIdentities, group);
} catch (UnknownIdentityException uie) {
- throw new AccountNotFoundException(String.format("Unable to set user group '%s': %s", StringUtils.join(newUserDns, ", "), uie.getMessage()), uie);
+ throw new AccountNotFoundException(String.format("Unable to set user group '%s': %s", StringUtils.join(newUserIdentities, ", "), uie.getMessage()), uie);
} catch (AuthorityAccessException aae) {
- throw new AdministrationException(String.format("Unable to set user group '%s': %s", StringUtils.join(newUserDns, ", "), aae.getMessage()), aae);
+ throw new AdministrationException(String.format("Unable to set user group '%s': %s", StringUtils.join(newUserIdentities, ", "), aae.getMessage()), aae);
}
}
@@ -118,35 +118,35 @@ public class UpdateUserGroupAction extends AbstractUserAction<Void> {
if (authorities != null) {
try {
// update the authority provider as approprivate
- authorityProvider.setAuthorities(user.getDn(), authorities);
+ authorityProvider.setAuthorities(user.getIdentity(), authorities);
// since all the authorities were updated accordingly, set the authorities
user.getAuthorities().clear();
user.getAuthorities().addAll(authorities);
} catch (UnknownIdentityException uie) {
- throw new AccountNotFoundException(String.format("Unable to modify authorities for '%s': %s.", user.getDn(), uie.getMessage()), uie);
+ throw new AccountNotFoundException(String.format("Unable to modify authorities for '%s': %s.", user.getIdentity(), uie.getMessage()), uie);
} catch (AuthorityAccessException aae) {
- throw new AdministrationException(String.format("Unable to access authorities for '%s': %s.", user.getDn(), aae.getMessage()), aae);
+ throw new AdministrationException(String.format("Unable to access authorities for '%s': %s.", user.getIdentity(), aae.getMessage()), aae);
}
} else {
try {
// refresh the authorities according to the provider
user.getAuthorities().clear();
- user.getAuthorities().addAll(authorityProvider.getAuthorities(user.getDn()));
+ user.getAuthorities().addAll(authorityProvider.getAuthorities(user.getIdentity()));
} catch (UnknownIdentityException uie) {
- throw new AccountNotFoundException(String.format("Unable to determine the authorities for '%s': %s.", user.getDn(), uie.getMessage()), uie);
+ throw new AccountNotFoundException(String.format("Unable to determine the authorities for '%s': %s.", user.getIdentity(), uie.getMessage()), uie);
} catch (AuthorityAccessException aae) {
- throw new AdministrationException(String.format("Unable to access authorities for '%s': %s.", user.getDn(), aae.getMessage()), aae);
+ throw new AdministrationException(String.format("Unable to access authorities for '%s': %s.", user.getIdentity(), aae.getMessage()), aae);
}
}
try {
// get the user group
- user.setUserGroup(authorityProvider.getGroupForUser(user.getDn()));
+ user.setUserGroup(authorityProvider.getGroupForUser(user.getIdentity()));
} catch (UnknownIdentityException uie) {
- throw new AccountNotFoundException(String.format("Unable to determine the group for '%s': %s.", user.getDn(), uie.getMessage()), uie);
+ throw new AccountNotFoundException(String.format("Unable to determine the group for '%s': %s.", user.getIdentity(), uie.getMessage()), uie);
} catch (AuthorityAccessException aae) {
- throw new AdministrationException(String.format("Unable to access the group for '%s': %s.", user.getDn(), aae.getMessage()), aae);
+ throw new AdministrationException(String.format("Unable to access the group for '%s': %s.", user.getIdentity(), aae.getMessage()), aae);
}
// update the users status in case they were previously pending or disabled
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/impl/StandardKeyService.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/impl/StandardKeyService.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/impl/StandardKeyService.java
new file mode 100644
index 0000000..ca0a124
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/service/impl/StandardKeyService.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin.service.impl;
+
+import org.apache.nifi.admin.dao.DataAccessException;
+import org.apache.nifi.admin.service.AdministrationException;
+import org.apache.nifi.admin.service.KeyService;
+import org.apache.nifi.admin.service.action.GetKeyByIdAction;
+import org.apache.nifi.admin.service.action.GetOrCreateKeyAction;
+import org.apache.nifi.admin.service.transaction.Transaction;
+import org.apache.nifi.admin.service.transaction.TransactionBuilder;
+import org.apache.nifi.admin.service.transaction.TransactionException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.nifi.key.Key;
+
+/**
+ *
+ */
+public class StandardKeyService implements KeyService {
+
+ private static final Logger logger = LoggerFactory.getLogger(StandardKeyService.class);
+
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+ private final ReentrantReadWriteLock.ReadLock readLock = lock.readLock();
+ private final ReentrantReadWriteLock.WriteLock writeLock = lock.writeLock();
+
+ private TransactionBuilder transactionBuilder;
+
+ @Override
+ public Key getKey(int id) {
+ Transaction transaction = null;
+ Key key = null;
+
+ readLock.lock();
+ try {
+ // start the transaction
+ transaction = transactionBuilder.start();
+
+ // get the key
+ GetKeyByIdAction addActions = new GetKeyByIdAction(id);
+ key = transaction.execute(addActions);
+
+ // commit the transaction
+ transaction.commit();
+ } catch (TransactionException | DataAccessException te) {
+ rollback(transaction);
+ throw new AdministrationException(te);
+ } catch (Throwable t) {
+ rollback(transaction);
+ throw t;
+ } finally {
+ closeQuietly(transaction);
+ readLock.unlock();
+ }
+
+ return key;
+ }
+
+ @Override
+ public Key getOrCreateKey(String identity) {
+ Transaction transaction = null;
+ Key key = null;
+
+ writeLock.lock();
+ try {
+ // start the transaction
+ transaction = transactionBuilder.start();
+
+ // seed the accounts
+ GetOrCreateKeyAction addActions = new GetOrCreateKeyAction(identity);
+ key = transaction.execute(addActions);
+
+ // commit the transaction
+ transaction.commit();
+ } catch (TransactionException | DataAccessException te) {
+ rollback(transaction);
+ throw new AdministrationException(te);
+ } catch (Throwable t) {
+ rollback(transaction);
+ throw t;
+ } finally {
+ closeQuietly(transaction);
+ writeLock.unlock();
+ }
+
+ return key;
+ }
+
+ private void rollback(Transaction transaction) {
+ if (transaction != null) {
+ transaction.rollback();
+ }
+ }
+
+ private void closeQuietly(final Transaction transaction) {
+ if (transaction != null) {
+ try {
+ transaction.close();
+ } catch (final IOException ioe) {
+ }
+ }
+ }
+
+ public void setTransactionBuilder(TransactionBuilder transactionBuilder) {
+ this.transactionBuilder = transactionBuilder;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/key/Key.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/key/Key.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/key/Key.java
new file mode 100644
index 0000000..9ce7a9a
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/key/Key.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.key;
+
+import java.io.Serializable;
+
+/**
+ * An signing key for a NiFi user.
+ */
+public class Key implements Serializable {
+
+ private int id;
+ private String identity;
+ private String key;
+
+ /**
+ * The key id.
+ *
+ * @return the id
+ */
+ public int getId() {
+ return id;
+ }
+
+ public void setId(int id) {
+ this.id = id;
+ }
+
+ /**
+ * The identity of the user this key is associated with.
+ *
+ * @return the identity
+ */
+ public String getIdentity() {
+ return identity;
+ }
+
+ public void setIdentity(String identity) {
+ this.identity = identity;
+ }
+
+ /**
+ * The signing key.
+ *
+ * @return the signing key
+ */
+ public String getKey() {
+ return key;
+ }
+
+ public void setKey(String key) {
+ this.key = key;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/user/NiFiUser.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/user/NiFiUser.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/user/NiFiUser.java
index a47bde9..231b133 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/user/NiFiUser.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/user/NiFiUser.java
@@ -29,10 +29,10 @@ import org.apache.commons.lang3.StringUtils;
*/
public class NiFiUser implements Serializable {
- public static final String ANONYMOUS_USER_DN = "anonymous";
+ public static final String ANONYMOUS_USER_IDENTITY = "anonymous";
private String id;
- private String dn;
+ private String identity;
private String userName;
private String userGroup;
private String justification;
@@ -55,12 +55,12 @@ public class NiFiUser implements Serializable {
this.creation = creation;
}
- public String getDn() {
- return dn;
+ public String getIdentity() {
+ return identity;
}
- public void setDn(String dn) {
- this.dn = dn;
+ public void setIdentity(String identity) {
+ this.identity = identity;
}
public String getUserName() {
@@ -143,7 +143,7 @@ public class NiFiUser implements Serializable {
return false;
}
final NiFiUser other = (NiFiUser) obj;
- if (!Objects.equals(this.dn, other.dn)) {
+ if (!Objects.equals(this.identity, other.identity)) {
return false;
}
return true;
@@ -152,13 +152,13 @@ public class NiFiUser implements Serializable {
@Override
public int hashCode() {
int hash = 7;
- hash = 53 * hash + Objects.hashCode(this.dn);
+ hash = 53 * hash + Objects.hashCode(this.identity);
return hash;
}
@Override
public String toString() {
- return String.format("dn[%s], userName[%s], justification[%s], authorities[%s]", getDn(), getUserName(), getJustification(), StringUtils.join(getAuthorities(), ", "));
+ return String.format("identity[%s], userName[%s], justification[%s], authorities[%s]", getIdentity(), getUserName(), getJustification(), StringUtils.join(getAuthorities(), ", "));
}
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/resources/nifi-administration-context.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/resources/nifi-administration-context.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/resources/nifi-administration-context.xml
index 8cb4b97..6d7b739 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/resources/nifi-administration-context.xml
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/resources/nifi-administration-context.xml
@@ -36,6 +36,11 @@
<bean id="auditDataSource" class="org.apache.nifi.admin.AuditDataSourceFactoryBean" destroy-method="shutdown" depends-on="userDataSource">
<property name="properties" ref="nifiProperties"/>
</bean>
+
+ <!-- initialize the data source -->
+ <bean id="keyDataSource" class="org.apache.nifi.admin.KeyDataSourceFactoryBean" destroy-method="shutdown">
+ <property name="properties" ref="nifiProperties"/>
+ </bean>
<!-- initialize the user transaction builder -->
<bean id="userTransactionBuilder" class="org.apache.nifi.admin.service.transaction.impl.StandardTransactionBuilder">
@@ -48,6 +53,12 @@
<property name="authorityProvider" ref="authorityProvider"/>
<property name="dataSource" ref="auditDataSource"/>
</bean>
+
+ <!-- initialize the key transaction builder -->
+ <bean id="keyTransactionBuilder" class="org.apache.nifi.admin.service.transaction.impl.StandardTransactionBuilder">
+ <property name="authorityProvider" ref="authorityProvider"/>
+ <property name="dataSource" ref="keyDataSource"/>
+ </bean>
<!-- administration service -->
<bean id="userService" class="org.apache.nifi.admin.service.impl.StandardUserService" init-method="seedUserAccounts">
@@ -59,4 +70,9 @@
<bean id="auditService" class="org.apache.nifi.admin.service.impl.StandardAuditService">
<property name="transactionBuilder" ref="auditTransactionBuilder"/>
</bean>
+
+ <!-- key service -->
+ <bean id="keyService" class="org.apache.nifi.admin.service.impl.StandardKeyService">
+ <property name="transactionBuilder" ref="keyTransactionBuilder"/>
+ </bean>
</beans>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/AuthorizeUserActionTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/AuthorizeUserActionTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/AuthorizeUserActionTest.java
index 28ea4a9..8d3c15a 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/AuthorizeUserActionTest.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/AuthorizeUserActionTest.java
@@ -54,17 +54,17 @@ public class AuthorizeUserActionTest {
private static final String USER_ID_10 = "10";
private static final String USER_ID_11 = "11";
- private static final String USER_DN_1 = "authority access exception while searching for user";
- private static final String USER_DN_2 = "unknown user";
- private static final String USER_DN_3 = "user removed after checking existence";
- private static final String USER_DN_4 = "access exception getting authorities";
- private static final String USER_DN_5 = "error creating user account";
- private static final String USER_DN_6 = "create user general sequence";
- private static final String USER_DN_7 = "existing user requires verification";
- private static final String USER_DN_8 = "existing user does not require verification";
- private static final String USER_DN_9 = "existing pending user";
- private static final String USER_DN_10 = "existing disabled user";
- private static final String USER_DN_11 = "existing user is now unknown in the authority provider";
+ private static final String USER_IDENTITY_1 = "authority access exception while searching for user";
+ private static final String USER_IDENTITY_2 = "unknown user";
+ private static final String USER_IDENTITY_3 = "user removed after checking existence";
+ private static final String USER_IDENTITY_4 = "access exception getting authorities";
+ private static final String USER_IDENTITY_5 = "error creating user account";
+ private static final String USER_IDENTITY_6 = "create user general sequence";
+ private static final String USER_IDENTITY_7 = "existing user requires verification";
+ private static final String USER_IDENTITY_8 = "existing user does not require verification";
+ private static final String USER_IDENTITY_9 = "existing pending user";
+ private static final String USER_IDENTITY_10 = "existing disabled user";
+ private static final String USER_IDENTITY_11 = "existing user is now unknown in the authority provider";
private DAOFactory daoFactory;
private UserDAO userDao;
@@ -85,18 +85,18 @@ public class AuthorizeUserActionTest {
if (USER_ID_7.equals(id)) {
user = new NiFiUser();
user.setId(USER_ID_7);
- user.setDn(USER_DN_7);
+ user.setIdentity(USER_IDENTITY_7);
user.getAuthorities().addAll(EnumSet.of(Authority.ROLE_MONITOR));
} else if (USER_ID_8.equals(id)) {
user = new NiFiUser();
user.setId(USER_ID_8);
- user.setDn(USER_DN_8);
+ user.setIdentity(USER_IDENTITY_8);
user.getAuthorities().addAll(EnumSet.of(Authority.ROLE_MONITOR));
user.setLastVerified(new Date());
} else if (USER_ID_11.equals(id)) {
user = new NiFiUser();
user.setId(USER_ID_11);
- user.setDn(USER_DN_11);
+ user.setIdentity(USER_IDENTITY_11);
user.getAuthorities().addAll(EnumSet.of(Authority.ROLE_MONITOR));
user.setStatus(AccountStatus.ACTIVE);
}
@@ -112,35 +112,35 @@ public class AuthorizeUserActionTest {
NiFiUser user = null;
switch (dn) {
- case USER_DN_7:
+ case USER_IDENTITY_7:
user = new NiFiUser();
user.setId(USER_ID_7);
- user.setDn(USER_DN_7);
+ user.setIdentity(USER_IDENTITY_7);
user.getAuthorities().addAll(EnumSet.of(Authority.ROLE_MONITOR));
break;
- case USER_DN_8:
+ case USER_IDENTITY_8:
user = new NiFiUser();
user.setId(USER_ID_8);
- user.setDn(USER_DN_8);
+ user.setIdentity(USER_IDENTITY_8);
user.getAuthorities().addAll(EnumSet.of(Authority.ROLE_MONITOR));
user.setLastVerified(new Date());
break;
- case USER_DN_9:
+ case USER_IDENTITY_9:
user = new NiFiUser();
user.setId(USER_ID_9);
- user.setDn(USER_DN_9);
+ user.setIdentity(USER_IDENTITY_9);
user.setStatus(AccountStatus.PENDING);
break;
- case USER_DN_10:
+ case USER_IDENTITY_10:
user = new NiFiUser();
user.setId(USER_ID_10);
- user.setDn(USER_DN_10);
+ user.setIdentity(USER_IDENTITY_10);
user.setStatus(AccountStatus.DISABLED);
break;
- case USER_DN_11:
+ case USER_IDENTITY_11:
user = new NiFiUser();
user.setId(USER_ID_11);
- user.setDn(USER_DN_11);
+ user.setIdentity(USER_IDENTITY_11);
user.getAuthorities().addAll(EnumSet.of(Authority.ROLE_MONITOR));
user.setStatus(AccountStatus.ACTIVE);
break;
@@ -154,10 +154,10 @@ public class AuthorizeUserActionTest {
public Void answer(InvocationOnMock invocation) throws Throwable {
Object[] args = invocation.getArguments();
NiFiUser user = (NiFiUser) args[0];
- switch (user.getDn()) {
- case USER_DN_5:
+ switch (user.getIdentity()) {
+ case USER_IDENTITY_5:
throw new DataAccessException();
- case USER_DN_6:
+ case USER_IDENTITY_6:
user.setId(USER_ID_6);
break;
}
@@ -215,9 +215,9 @@ public class AuthorizeUserActionTest {
Object[] args = invocation.getArguments();
String dn = (String) args[0];
switch (dn) {
- case USER_DN_1:
+ case USER_IDENTITY_1:
throw new AuthorityAccessException(StringUtils.EMPTY);
- case USER_DN_2:
+ case USER_IDENTITY_2:
return false;
}
@@ -231,21 +231,21 @@ public class AuthorizeUserActionTest {
String dn = (String) args[0];
Set<Authority> authorities = EnumSet.noneOf(Authority.class);
switch (dn) {
- case USER_DN_3:
+ case USER_IDENTITY_3:
throw new UnknownIdentityException(StringUtils.EMPTY);
- case USER_DN_4:
+ case USER_IDENTITY_4:
throw new AuthorityAccessException(StringUtils.EMPTY);
- case USER_DN_6:
+ case USER_IDENTITY_6:
authorities.add(Authority.ROLE_MONITOR);
break;
- case USER_DN_7:
+ case USER_IDENTITY_7:
authorities.add(Authority.ROLE_DFM);
break;
- case USER_DN_9:
+ case USER_IDENTITY_9:
throw new UnknownIdentityException(StringUtils.EMPTY);
- case USER_DN_10:
+ case USER_IDENTITY_10:
throw new UnknownIdentityException(StringUtils.EMPTY);
- case USER_DN_11:
+ case USER_IDENTITY_11:
throw new UnknownIdentityException(StringUtils.EMPTY);
}
@@ -272,7 +272,7 @@ public class AuthorizeUserActionTest {
*/
@Test(expected = AdministrationException.class)
public void testAuthorityAccessExceptionInDoesDnExist() throws Exception {
- AuthorizeUserAction authorizeUser = new AuthorizeUserAction(USER_DN_1, 0);
+ AuthorizeUserAction authorizeUser = new AuthorizeUserAction(USER_IDENTITY_1, 0);
authorizeUser.execute(daoFactory, authorityProvider);
}
@@ -283,7 +283,7 @@ public class AuthorizeUserActionTest {
*/
@Test(expected = AccountNotFoundException.class)
public void testUnknownUser() throws Exception {
- AuthorizeUserAction authorizeUser = new AuthorizeUserAction(USER_DN_2, 0);
+ AuthorizeUserAction authorizeUser = new AuthorizeUserAction(USER_IDENTITY_2, 0);
authorizeUser.execute(daoFactory, authorityProvider);
}
@@ -294,7 +294,7 @@ public class AuthorizeUserActionTest {
*/
@Test(expected = AccountNotFoundException.class)
public void testUserRemovedAfterCheckingExistence() throws Exception {
- AuthorizeUserAction authorizeUser = new AuthorizeUserAction(USER_DN_3, 0);
+ AuthorizeUserAction authorizeUser = new AuthorizeUserAction(USER_IDENTITY_3, 0);
authorizeUser.execute(daoFactory, authorityProvider);
}
@@ -305,7 +305,7 @@ public class AuthorizeUserActionTest {
*/
@Test(expected = AdministrationException.class)
public void testAuthorityAccessException() throws Exception {
- AuthorizeUserAction authorizeUser = new AuthorizeUserAction(USER_DN_4, 0);
+ AuthorizeUserAction authorizeUser = new AuthorizeUserAction(USER_IDENTITY_4, 0);
authorizeUser.execute(daoFactory, authorityProvider);
}
@@ -316,7 +316,7 @@ public class AuthorizeUserActionTest {
*/
@Test(expected = DataAccessException.class)
public void testErrorCreatingUserAccount() throws Exception {
- AuthorizeUserAction authorizeUser = new AuthorizeUserAction(USER_DN_5, 0);
+ AuthorizeUserAction authorizeUser = new AuthorizeUserAction(USER_IDENTITY_5, 0);
authorizeUser.execute(daoFactory, authorityProvider);
}
@@ -327,11 +327,11 @@ public class AuthorizeUserActionTest {
*/
@Test
public void testAccountCreation() throws Exception {
- AuthorizeUserAction authorizeUser = new AuthorizeUserAction(USER_DN_6, 0);
+ AuthorizeUserAction authorizeUser = new AuthorizeUserAction(USER_IDENTITY_6, 0);
NiFiUser user = authorizeUser.execute(daoFactory, authorityProvider);
// verify the user
- Assert.assertEquals(USER_DN_6, user.getDn());
+ Assert.assertEquals(USER_IDENTITY_6, user.getIdentity());
Assert.assertEquals(1, user.getAuthorities().size());
Assert.assertTrue(user.getAuthorities().contains(Authority.ROLE_MONITOR));
@@ -347,11 +347,11 @@ public class AuthorizeUserActionTest {
*/
@Test
public void testExistingUserRequiresVerification() throws Exception {
- AuthorizeUserAction authorizeUser = new AuthorizeUserAction(USER_DN_7, 0);
+ AuthorizeUserAction authorizeUser = new AuthorizeUserAction(USER_IDENTITY_7, 0);
NiFiUser user = authorizeUser.execute(daoFactory, authorityProvider);
// verify the user
- Assert.assertEquals(USER_DN_7, user.getDn());
+ Assert.assertEquals(USER_IDENTITY_7, user.getIdentity());
Assert.assertEquals(1, user.getAuthorities().size());
Assert.assertTrue(user.getAuthorities().contains(Authority.ROLE_DFM));
@@ -369,11 +369,11 @@ public class AuthorizeUserActionTest {
@Test
public void testExistingUserNoVerification() throws Exception {
// disabling verification by passing in a large cache duration
- AuthorizeUserAction authorizeUser = new AuthorizeUserAction(USER_DN_8, Integer.MAX_VALUE);
+ AuthorizeUserAction authorizeUser = new AuthorizeUserAction(USER_IDENTITY_8, Integer.MAX_VALUE);
NiFiUser user = authorizeUser.execute(daoFactory, authorityProvider);
// verify the user
- Assert.assertEquals(USER_DN_8, user.getDn());
+ Assert.assertEquals(USER_IDENTITY_8, user.getIdentity());
Assert.assertEquals(1, user.getAuthorities().size());
Assert.assertTrue(user.getAuthorities().contains(Authority.ROLE_MONITOR));
@@ -391,7 +391,7 @@ public class AuthorizeUserActionTest {
@Test(expected = AccountPendingException.class)
public void testExistingPendingUser() throws Exception {
// disabling verification by passing in a large cache duration
- AuthorizeUserAction authorizeUser = new AuthorizeUserAction(USER_DN_9, Integer.MAX_VALUE);
+ AuthorizeUserAction authorizeUser = new AuthorizeUserAction(USER_IDENTITY_9, Integer.MAX_VALUE);
authorizeUser.execute(daoFactory, authorityProvider);
}
@@ -403,7 +403,7 @@ public class AuthorizeUserActionTest {
@Test(expected = AccountDisabledException.class)
public void testExistingDisabledUser() throws Exception {
// disabling verification by passing in a large cache duration
- AuthorizeUserAction authorizeUser = new AuthorizeUserAction(USER_DN_10, Integer.MAX_VALUE);
+ AuthorizeUserAction authorizeUser = new AuthorizeUserAction(USER_IDENTITY_10, Integer.MAX_VALUE);
authorizeUser.execute(daoFactory, authorityProvider);
}
@@ -416,7 +416,7 @@ public class AuthorizeUserActionTest {
@Test
public void testExistingActiveUserNotFoundInProvider() throws Exception {
try {
- AuthorizeUserAction authorizeUser = new AuthorizeUserAction(USER_DN_11, 0);
+ AuthorizeUserAction authorizeUser = new AuthorizeUserAction(USER_IDENTITY_11, 0);
authorizeUser.execute(daoFactory, authorityProvider);
Assert.fail();
[14/50] [abbrv] nifi git commit: NIFI-1175 Exposing minimum
properties required to create an HBase connection on the HBaseClientService
as an optional alternative to the conf files
Posted by mc...@apache.org.
NIFI-1175 Exposing minimum properties required to create an HBase connection on the HBaseClientService as an optional alternative to the conf files
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/2b9b5e00
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/2b9b5e00
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/2b9b5e00
Branch: refs/heads/NIFI-655
Commit: 2b9b5e008f307ff5f31c36840ec503cacae5ff30
Parents: 453b140
Author: Bryan Bende <bb...@apache.org>
Authored: Mon Nov 16 14:01:23 2015 -0500
Committer: Bryan Bende <bb...@apache.org>
Committed: Tue Nov 17 12:01:46 2015 -0500
----------------------------------------------------------------------
.../apache/nifi/hbase/HBaseClientService.java | 30 ++++++-
.../nifi-hbase_1_1_2-client-service/pom.xml | 6 ++
.../nifi/hbase/HBase_1_1_2_ClientService.java | 93 +++++++++++++++++++-
.../hbase/TestHBase_1_1_2_ClientService.java | 67 ++++++++++++++
4 files changed, 190 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/2b9b5e00/nifi-nar-bundles/nifi-standard-services/nifi-hbase-client-service-api/src/main/java/org/apache/nifi/hbase/HBaseClientService.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-hbase-client-service-api/src/main/java/org/apache/nifi/hbase/HBaseClientService.java b/nifi-nar-bundles/nifi-standard-services/nifi-hbase-client-service-api/src/main/java/org/apache/nifi/hbase/HBaseClientService.java
index a3f2040..9ff2c46 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-hbase-client-service-api/src/main/java/org/apache/nifi/hbase/HBaseClientService.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-hbase-client-service-api/src/main/java/org/apache/nifi/hbase/HBaseClientService.java
@@ -24,6 +24,7 @@ import org.apache.nifi.hbase.put.PutFlowFile;
import org.apache.nifi.hbase.scan.Column;
import org.apache.nifi.hbase.scan.ResultHandler;
import org.apache.nifi.hbase.validate.ConfigFilesValidator;
+import org.apache.nifi.processor.util.StandardValidators;
import java.io.IOException;
import java.util.Collection;
@@ -34,12 +35,35 @@ public interface HBaseClientService extends ControllerService {
PropertyDescriptor HADOOP_CONF_FILES = new PropertyDescriptor.Builder()
.name("Hadoop Configuration Files")
- .description("Comma-separated list of Hadoop Configuration files, such as hbase-site.xml")
- .required(true)
- .defaultValue("./conf/hbase-site.xml")
+ .description("Comma-separated list of Hadoop Configuration files, such as hbase-site.xml, including full paths to the files.")
.addValidator(new ConfigFilesValidator())
.build();
+ PropertyDescriptor ZOOKEEPER_QUORUM = new PropertyDescriptor.Builder()
+ .name("ZooKeeper Quorum")
+ .description("Comma-separated list of ZooKeeper hosts for HBase. Required if Hadoop Configuration Files are not provided.")
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+
+ PropertyDescriptor ZOOKEEPER_CLIENT_PORT = new PropertyDescriptor.Builder()
+ .name("ZooKeeper Client Port")
+ .description("The port on which ZooKeeper is accepting client connections. Required if Hadoop Configuration Files are not provided.")
+ .addValidator(StandardValidators.PORT_VALIDATOR)
+ .build();
+
+ PropertyDescriptor ZOOKEEPER_ZNODE_PARENT = new PropertyDescriptor.Builder()
+ .name("ZooKeeper ZNode Parent")
+ .description("The ZooKeeper ZNode Parent value for HBase (example: /hbase). Required if Hadoop Configuration Files are not provided.")
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+
+ PropertyDescriptor HBASE_CLIENT_RETRIES = new PropertyDescriptor.Builder()
+ .name("HBase Client Retries")
+ .description("The number of times the HBase client will retry connecting. Required if Hadoop Configuration Files are not provided.")
+ .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR)
+ .defaultValue("1")
+ .build();
+
/**
* Puts a batch of mutations to the given table.
*
http://git-wip-us.apache.org/repos/asf/nifi/blob/2b9b5e00/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/pom.xml b/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/pom.xml
index a90e0e3..68ea55d 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/pom.xml
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/pom.xml
@@ -57,6 +57,12 @@
<artifactId>commons-lang3</artifactId>
<version>3.4</version>
</dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>log4j-over-slf4j</artifactId>
+ <version>${org.slf4j.version}</version>
+ </dependency>
+
<!-- test dependencies -->
<dependency>
http://git-wip-us.apache.org/repos/asf/nifi/blob/2b9b5e00/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/main/java/org/apache/nifi/hbase/HBase_1_1_2_ClientService.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/main/java/org/apache/nifi/hbase/HBase_1_1_2_ClientService.java b/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/main/java/org/apache/nifi/hbase/HBase_1_1_2_ClientService.java
index 9c300db..42590c2 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/main/java/org/apache/nifi/hbase/HBase_1_1_2_ClientService.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/main/java/org/apache/nifi/hbase/HBase_1_1_2_ClientService.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Put;
@@ -31,11 +32,14 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.ParseFilter;
+import org.apache.nifi.annotation.behavior.DynamicProperty;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.annotation.lifecycle.OnDisabled;
import org.apache.nifi.annotation.lifecycle.OnEnabled;
import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
import org.apache.nifi.controller.AbstractControllerService;
import org.apache.nifi.controller.ConfigurationContext;
import org.apache.nifi.controller.ControllerServiceInitializationContext;
@@ -43,6 +47,7 @@ import org.apache.nifi.hbase.put.PutFlowFile;
import org.apache.nifi.hbase.scan.Column;
import org.apache.nifi.hbase.scan.ResultCell;
import org.apache.nifi.hbase.scan.ResultHandler;
+import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.reporting.InitializationException;
import java.io.IOException;
@@ -55,9 +60,20 @@ import java.util.List;
import java.util.Map;
@Tags({ "hbase", "client"})
-@CapabilityDescription("Implementation of HBaseClientService for HBase 1.1.2.")
+@CapabilityDescription("Implementation of HBaseClientService for HBase 1.1.2. This service can be configured by providing " +
+ "a comma-separated list of configuration files, or by specifying values for the other properties. If configuration files " +
+ "are provided, they will be loaded first, and the values of the additional properties will override the values from " +
+ "the configuration files. In addition, any user defined properties on the processor will also be passed to the HBase " +
+ "configuration.")
+@DynamicProperty(name="The name of an HBase configuration property.", value="The value of the given HBase configuration property.",
+ description="These properties will be set on the HBase configuration after loading any provided configuration files.")
public class HBase_1_1_2_ClientService extends AbstractControllerService implements HBaseClientService {
+ static final String HBASE_CONF_ZK_QUORUM = "hbase.zookeeper.quorum";
+ static final String HBASE_CONF_ZK_PORT = "hbase.zookeeper.property.clientPort";
+ static final String HBASE_CONF_ZNODE_PARENT = "zookeeper.znode.parent";
+ static final String HBASE_CONF_CLIENT_RETRIES = "hbase.client.retries.number";
+
private volatile Connection connection;
private List<PropertyDescriptor> properties;
@@ -65,6 +81,10 @@ public class HBase_1_1_2_ClientService extends AbstractControllerService impleme
protected void init(ControllerServiceInitializationContext config) throws InitializationException {
List<PropertyDescriptor> props = new ArrayList<>();
props.add(HADOOP_CONF_FILES);
+ props.add(ZOOKEEPER_QUORUM);
+ props.add(ZOOKEEPER_CLIENT_PORT);
+ props.add(ZOOKEEPER_ZNODE_PARENT);
+ props.add(HBASE_CLIENT_RETRIES);
this.properties = Collections.unmodifiableList(props);
}
@@ -73,16 +93,83 @@ public class HBase_1_1_2_ClientService extends AbstractControllerService impleme
return properties;
}
+ @Override
+ protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(String propertyDescriptorName) {
+ return new PropertyDescriptor.Builder()
+ .description("Specifies the value for '" + propertyDescriptorName + "' in the HBase configuration.")
+ .name(propertyDescriptorName)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .dynamic(true)
+ .build();
+ }
+
+ @Override
+ protected Collection<ValidationResult> customValidate(ValidationContext validationContext) {
+ boolean confFileProvided = validationContext.getProperty(HADOOP_CONF_FILES).isSet();
+ boolean zkQuorumProvided = validationContext.getProperty(ZOOKEEPER_QUORUM).isSet();
+ boolean zkPortProvided = validationContext.getProperty(ZOOKEEPER_CLIENT_PORT).isSet();
+ boolean znodeParentProvided = validationContext.getProperty(ZOOKEEPER_ZNODE_PARENT).isSet();
+ boolean retriesProvided = validationContext.getProperty(HBASE_CLIENT_RETRIES).isSet();
+
+ final List<ValidationResult> problems = new ArrayList<>();
+
+ if (!confFileProvided && (!zkQuorumProvided || !zkPortProvided || !znodeParentProvided || !retriesProvided)) {
+ problems.add(new ValidationResult.Builder()
+ .valid(false)
+ .subject(this.getClass().getSimpleName())
+ .explanation("ZooKeeper Quorum, ZooKeeper Client Port, ZooKeeper ZNode Parent, and HBase Client Retries are required " +
+ "when Hadoop Configuration Files are not provided.")
+ .build());
+ }
+
+ return problems;
+ }
+
@OnEnabled
public void onEnabled(final ConfigurationContext context) throws InitializationException, IOException {
this.connection = createConnection(context);
+
+ // connection check
+ if (this.connection != null) {
+ final Admin admin = this.connection.getAdmin();
+ if (admin != null) {
+ admin.listTableNames();
+ }
+ }
}
protected Connection createConnection(final ConfigurationContext context) throws IOException {
final Configuration hbaseConfig = HBaseConfiguration.create();
- for (final String configFile : context.getProperty(HADOOP_CONF_FILES).getValue().split(",")) {
- hbaseConfig.addResource(new Path(configFile.trim()));
+
+ // if conf files are provided, start with those
+ if (context.getProperty(HADOOP_CONF_FILES).isSet()) {
+ for (final String configFile : context.getProperty(HADOOP_CONF_FILES).getValue().split(",")) {
+ hbaseConfig.addResource(new Path(configFile.trim()));
+ }
+ }
+
+ // override with any properties that are provided
+ if (context.getProperty(ZOOKEEPER_QUORUM).isSet()) {
+ hbaseConfig.set(HBASE_CONF_ZK_QUORUM, context.getProperty(ZOOKEEPER_QUORUM).getValue());
+ }
+ if (context.getProperty(ZOOKEEPER_CLIENT_PORT).isSet()) {
+ hbaseConfig.set(HBASE_CONF_ZK_PORT, context.getProperty(ZOOKEEPER_CLIENT_PORT).getValue());
}
+ if (context.getProperty(ZOOKEEPER_ZNODE_PARENT).isSet()) {
+ hbaseConfig.set(HBASE_CONF_ZNODE_PARENT, context.getProperty(ZOOKEEPER_ZNODE_PARENT).getValue());
+ }
+ if (context.getProperty(HBASE_CLIENT_RETRIES).isSet()) {
+ hbaseConfig.set(HBASE_CONF_CLIENT_RETRIES, context.getProperty(HBASE_CLIENT_RETRIES).getValue());
+ }
+
+ // add any dynamic properties to the HBase configuration
+ for (final Map.Entry<PropertyDescriptor, String> entry : context.getProperties().entrySet()) {
+ final PropertyDescriptor descriptor = entry.getKey();
+ if (descriptor.isDynamic()) {
+ hbaseConfig.set(descriptor.getName(), entry.getValue());
+ }
+ }
+
return ConnectionFactory.createConnection(hbaseConfig);
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/2b9b5e00/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/test/java/org/apache/nifi/hbase/TestHBase_1_1_2_ClientService.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/test/java/org/apache/nifi/hbase/TestHBase_1_1_2_ClientService.java b/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/test/java/org/apache/nifi/hbase/TestHBase_1_1_2_ClientService.java
index 71dd51b..1575f3c 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/test/java/org/apache/nifi/hbase/TestHBase_1_1_2_ClientService.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/test/java/org/apache/nifi/hbase/TestHBase_1_1_2_ClientService.java
@@ -56,6 +56,73 @@ import static org.mockito.Mockito.when;
public class TestHBase_1_1_2_ClientService {
@Test
+ public void testCustomValidate() throws InitializationException {
+ final TestRunner runner = TestRunners.newTestRunner(TestProcessor.class);
+
+ final String tableName = "nifi";
+ final Table table = Mockito.mock(Table.class);
+ when(table.getName()).thenReturn(TableName.valueOf(tableName));
+
+ // no conf file or zk properties so should be invalid
+ MockHBaseClientService service = new MockHBaseClientService(table);
+ runner.addControllerService("hbaseClientService", service);
+ runner.enableControllerService(service);
+
+ runner.assertNotValid(service);
+ runner.removeControllerService(service);
+
+ // conf file with no zk properties should be valid
+ service = new MockHBaseClientService(table);
+ runner.addControllerService("hbaseClientService", service);
+ runner.setProperty(service, HBase_1_1_2_ClientService.HADOOP_CONF_FILES, "src/test/resources/core-site.xml");
+ runner.enableControllerService(service);
+
+ runner.assertValid(service);
+ runner.removeControllerService(service);
+
+ // only quorum and no conf file should be invalid
+ service = new MockHBaseClientService(table);
+ runner.addControllerService("hbaseClientService", service);
+ runner.setProperty(service, HBase_1_1_2_ClientService.ZOOKEEPER_QUORUM, "localhost");
+ runner.enableControllerService(service);
+
+ runner.assertNotValid(service);
+ runner.removeControllerService(service);
+
+ // quorum and port, no znode, no conf file, should be invalid
+ service = new MockHBaseClientService(table);
+ runner.addControllerService("hbaseClientService", service);
+ runner.setProperty(service, HBase_1_1_2_ClientService.ZOOKEEPER_QUORUM, "localhost");
+ runner.setProperty(service, HBase_1_1_2_ClientService.ZOOKEEPER_CLIENT_PORT, "2181");
+ runner.enableControllerService(service);
+
+ runner.assertNotValid(service);
+ runner.removeControllerService(service);
+
+ // quorum, port, and znode, no conf file, should be valid
+ service = new MockHBaseClientService(table);
+ runner.addControllerService("hbaseClientService", service);
+ runner.setProperty(service, HBase_1_1_2_ClientService.ZOOKEEPER_QUORUM, "localhost");
+ runner.setProperty(service, HBase_1_1_2_ClientService.ZOOKEEPER_CLIENT_PORT, "2181");
+ runner.setProperty(service, HBase_1_1_2_ClientService.ZOOKEEPER_ZNODE_PARENT, "/hbase");
+ runner.enableControllerService(service);
+
+ runner.assertValid(service);
+ runner.removeControllerService(service);
+
+ // quorum and port with conf file should be valid
+ service = new MockHBaseClientService(table);
+ runner.addControllerService("hbaseClientService", service);
+ runner.setProperty(service, HBase_1_1_2_ClientService.HADOOP_CONF_FILES, "src/test/resources/core-site.xml");
+ runner.setProperty(service, HBase_1_1_2_ClientService.ZOOKEEPER_QUORUM, "localhost");
+ runner.setProperty(service, HBase_1_1_2_ClientService.ZOOKEEPER_CLIENT_PORT, "2181");
+ runner.enableControllerService(service);
+
+ runner.assertValid(service);
+ runner.removeControllerService(service);
+ }
+
+ @Test
public void testSinglePut() throws InitializationException, IOException {
final String tableName = "nifi";
final String row = "row1";
[08/50] [abbrv] nifi git commit: NIFI-748 addressed PR comments -
made DocReader package private - polished logic in read(..) method to avoid
escaping the loop - added call to sorting logic in
LuceneUtil.groupDocsByStorageFileName(..) to ensure that prev
Posted by mc...@apache.org.
NIFI-748 addressed PR comments
- made DocReader package private
- polished logic in read(..) method to avoid escaping the loop
- added call to sorting logic in LuceneUtil.groupDocsByStorageFileName(..) to ensure that previous behavior and assumptions in read(..) methodd are preserved
- other minor polishing
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/15880f9f
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/15880f9f
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/15880f9f
Branch: refs/heads/NIFI-655
Commit: 15880f9fcc78e7a040e0d64d2a517390122e3fe3
Parents: a4d93c6
Author: Oleg Zhurakousky <ol...@suitcase.io>
Authored: Mon Nov 16 08:39:23 2015 -0500
Committer: Oleg Zhurakousky <ol...@suitcase.io>
Committed: Mon Nov 16 08:39:23 2015 -0500
----------------------------------------------------------------------
.../nifi/provenance/lucene/DocsReader.java | 21 ++++++++++----------
.../nifi/provenance/lucene/LuceneUtil.java | 15 ++++++++++++--
2 files changed, 24 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/15880f9f/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/DocsReader.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/DocsReader.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/DocsReader.java
index 7bd800b..703a5b8 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/DocsReader.java
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/DocsReader.java
@@ -22,6 +22,7 @@ import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
@@ -43,7 +44,7 @@ import org.apache.lucene.search.TopDocs;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class DocsReader {
+class DocsReader {
private final Logger logger = LoggerFactory.getLogger(DocsReader.class);
public Set<ProvenanceEventRecord> read(final TopDocs topDocs, final IndexReader indexReader, final Collection<Path> allProvenanceLogFiles,
@@ -106,12 +107,13 @@ public class DocsReader {
public Set<ProvenanceEventRecord> read(final List<Document> docs, final Collection<Path> allProvenanceLogFiles,
final AtomicInteger retrievalCount, final int maxResults, final int maxAttributeChars) throws IOException {
+ if (retrievalCount.get() >= maxResults) {
+ return Collections.emptySet();
+ }
+
final long start = System.nanoTime();
Set<ProvenanceEventRecord> matchingRecords = new LinkedHashSet<>();
- if (retrievalCount.get() >= maxResults) {
- return matchingRecords;
- }
Map<String, List<Document>> byStorageNameDocGroups = LuceneUtil.groupDocsByStorageFileName(docs);
@@ -123,17 +125,16 @@ public class DocsReader {
if (provenanceEventFile != null) {
try (RecordReader reader = RecordReaders.newRecordReader(provenanceEventFile, allProvenanceLogFiles,
maxAttributeChars)) {
- for (Document document : byStorageNameDocGroups.get(storageFileName)) {
- ProvenanceEventRecord eRec = this.getRecord(document, reader);
+
+ Iterator<Document> docIter = byStorageNameDocGroups.get(storageFileName).iterator();
+ while (docIter.hasNext() && retrievalCount.incrementAndGet() < maxResults){
+ ProvenanceEventRecord eRec = this.getRecord(docIter.next(), reader);
if (eRec != null) {
matchingRecords.add(eRec);
eventsReadThisFile++;
-
- if (retrievalCount.incrementAndGet() >= maxResults) {
- break;
- }
}
}
+
} catch (Exception e) {
logger.warn("Failed while trying to read Provenance Events. The event file '"
+ provenanceEventFile.getAbsolutePath() +
http://git-wip-us.apache.org/repos/asf/nifi/blob/15880f9f/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/LuceneUtil.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/LuceneUtil.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/LuceneUtil.java
index aa50b94..08a99d6 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/LuceneUtil.java
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/LuceneUtil.java
@@ -130,8 +130,14 @@ public class LuceneUtil {
return luceneQuery;
}
+ /**
+ * Will sort documents by filename and then file offset so that we can
+ * retrieve the records efficiently
+ *
+ * @param documents
+ * list of {@link Document}s
+ */
public static void sortDocsForRetrieval(final List<Document> documents) {
- // sort by filename and then file offset so that we can retrieve the records efficiently
Collections.sort(documents, new Comparator<Document>() {
@Override
public int compare(final Document o1, final Document o2) {
@@ -167,7 +173,9 @@ public class LuceneUtil {
* Will group documents based on the {@link FieldNames#STORAGE_FILENAME}.
*
* @param documents
- * list of {@link Document}s
+ * list of {@link Document}s which will be sorted via
+ * {@link #sortDocsForRetrieval(List)} for more efficient record
+ * retrieval.
* @return a {@link Map} of document groups with
* {@link FieldNames#STORAGE_FILENAME} as key and {@link List} of
* {@link Document}s as value.
@@ -181,6 +189,9 @@ public class LuceneUtil {
}
documentGroups.get(fileName).add(document);
}
+ for (List<Document> groupedDocuments : documentGroups.values()) {
+ sortDocsForRetrieval(groupedDocuments);
+ }
return documentGroups;
}
}
[19/50] [abbrv] nifi git commit: NIFI-1108 Providing additional
annotations on processors to ensure utilization of the InputRequirement
annotation.
Posted by mc...@apache.org.
NIFI-1108 Providing additional annotations on processors to ensure utilization of the InputRequirement annotation.
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/773576e0
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/773576e0
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/773576e0
Branch: refs/heads/NIFI-655
Commit: 773576e041088d9e326f1d2e84b0ad8acbd6cfdc
Parents: 911e1c5
Author: Aldrin Piri <al...@apache.org>
Authored: Wed Nov 18 13:56:21 2015 -0500
Committer: Aldrin Piri <al...@apache.org>
Committed: Wed Nov 18 13:56:21 2015 -0500
----------------------------------------------------------------------
.../java/org/apache/nifi/processors/aws/s3/DeleteS3Object.java | 3 +++
.../apache/nifi/processors/azure/eventhub/GetAzureEventHub.java | 3 +++
.../apache/nifi/processors/azure/eventhub/PutAzureEventHub.java | 3 +++
.../org/apache/nifi/processors/couchbase/PutCouchbaseKey.java | 3 +++
.../java/org/apache/nifi/processors/kite/ConvertAvroSchema.java | 3 +++
.../main/java/org/apache/nifi/processors/mongodb/GetMongo.java | 3 +++
.../main/java/org/apache/nifi/processors/mongodb/PutMongo.java | 3 +++
.../nifi/processors/standard/EvaluateRegularExpression.java | 3 +++
8 files changed, 24 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/773576e0/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/DeleteS3Object.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/DeleteS3Object.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/DeleteS3Object.java
index 3d2a9a6..6b078d1 100644
--- a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/DeleteS3Object.java
+++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/DeleteS3Object.java
@@ -26,6 +26,8 @@ import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.DeleteObjectRequest;
import com.amazonaws.services.s3.model.DeleteVersionRequest;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
import org.apache.nifi.annotation.behavior.SupportsBatching;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.SeeAlso;
@@ -40,6 +42,7 @@ import org.apache.nifi.processor.util.StandardValidators;
@SupportsBatching
@SeeAlso({PutS3Object.class})
@Tags({"Amazon", "S3", "AWS", "Archive", "Delete"})
+@InputRequirement(Requirement.INPUT_REQUIRED)
@CapabilityDescription("Deletes FlowFiles on an Amazon S3 Bucket. " +
"If attempting to delete a file that does not exist, FlowFile is routed to success.")
public class DeleteS3Object extends AbstractS3Processor {
http://git-wip-us.apache.org/repos/asf/nifi/blob/773576e0/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/eventhub/GetAzureEventHub.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/eventhub/GetAzureEventHub.java b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/eventhub/GetAzureEventHub.java
index 2c62df6..ffd0bab 100644
--- a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/eventhub/GetAzureEventHub.java
+++ b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/eventhub/GetAzureEventHub.java
@@ -30,6 +30,8 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
import org.apache.nifi.annotation.behavior.WritesAttribute;
import org.apache.nifi.annotation.behavior.WritesAttributes;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
@@ -56,6 +58,7 @@ import com.microsoft.eventhubs.client.ResilientEventHubReceiver;
@Tags({ "azure", "microsoft", "cloud", "eventhub", "events", "streaming", "streams" })
@CapabilityDescription("Receives messages from a Microsoft Azure Event Hub, writing the contents of the Azure message to the content of the FlowFile")
+@InputRequirement(Requirement.INPUT_FORBIDDEN)
@WritesAttributes({
@WritesAttribute(attribute = "eventhub.enqueued.timestamp", description = "The time (in milliseconds since epoch, UTC) at which the message was enqueued in the Azure Event Hub"),
@WritesAttribute(attribute = "eventhub.offset", description = "The offset into the partition at which the message was stored"),
http://git-wip-us.apache.org/repos/asf/nifi/blob/773576e0/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/eventhub/PutAzureEventHub.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/eventhub/PutAzureEventHub.java b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/eventhub/PutAzureEventHub.java
index 82fff27..3bf4ddc 100644
--- a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/eventhub/PutAzureEventHub.java
+++ b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/eventhub/PutAzureEventHub.java
@@ -28,6 +28,8 @@ import java.util.concurrent.TimeUnit;
import javax.jms.DeliveryMode;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
import org.apache.nifi.annotation.behavior.SupportsBatching;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
@@ -53,6 +55,7 @@ import com.microsoft.eventhubs.client.EventHubSender;
@SupportsBatching
@Tags({ "microsoft", "azure", "cloud", "eventhub", "events", "streams", "streaming" })
+@InputRequirement(Requirement.INPUT_REQUIRED)
@CapabilityDescription("Sends the contents of a FlowFile to a Windows Azure Event Hub. Note: the content of the FlowFile will be buffered into memory before being sent, "
+ "so care should be taken to avoid sending FlowFiles to this Processor that exceed the amount of Java Heap Space available.")
public class PutAzureEventHub extends AbstractProcessor {
http://git-wip-us.apache.org/repos/asf/nifi/blob/773576e0/nifi-nar-bundles/nifi-couchbase-bundle/nifi-couchbase-processors/src/main/java/org/apache/nifi/processors/couchbase/PutCouchbaseKey.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-couchbase-bundle/nifi-couchbase-processors/src/main/java/org/apache/nifi/processors/couchbase/PutCouchbaseKey.java b/nifi-nar-bundles/nifi-couchbase-bundle/nifi-couchbase-processors/src/main/java/org/apache/nifi/processors/couchbase/PutCouchbaseKey.java
index 291c02c..b03f3f2 100644
--- a/nifi-nar-bundles/nifi-couchbase-bundle/nifi-couchbase-processors/src/main/java/org/apache/nifi/processors/couchbase/PutCouchbaseKey.java
+++ b/nifi-nar-bundles/nifi-couchbase-bundle/nifi-couchbase-processors/src/main/java/org/apache/nifi/processors/couchbase/PutCouchbaseKey.java
@@ -25,6 +25,8 @@ import java.util.Map;
import java.util.Set;
import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
import org.apache.nifi.annotation.behavior.ReadsAttribute;
import org.apache.nifi.annotation.behavior.ReadsAttributes;
import org.apache.nifi.annotation.behavior.WritesAttribute;
@@ -57,6 +59,7 @@ import com.couchbase.client.java.document.RawJsonDocument;
@Tags({"nosql", "couchbase", "database", "put"})
@CapabilityDescription("Put a document to Couchbase Server via Key/Value access.")
@SeeAlso({CouchbaseClusterControllerService.class})
+@InputRequirement(Requirement.INPUT_REQUIRED)
@ReadsAttributes({
@ReadsAttribute(attribute = "uuid", description = "Used as a document id if 'Document Id' is not specified"),
})
http://git-wip-us.apache.org/repos/asf/nifi/blob/773576e0/nifi-nar-bundles/nifi-kite-bundle/nifi-kite-processors/src/main/java/org/apache/nifi/processors/kite/ConvertAvroSchema.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-kite-bundle/nifi-kite-processors/src/main/java/org/apache/nifi/processors/kite/ConvertAvroSchema.java b/nifi-nar-bundles/nifi-kite-bundle/nifi-kite-processors/src/main/java/org/apache/nifi/processors/kite/ConvertAvroSchema.java
index daeb548..d64f5df 100644
--- a/nifi-nar-bundles/nifi-kite-bundle/nifi-kite-processors/src/main/java/org/apache/nifi/processors/kite/ConvertAvroSchema.java
+++ b/nifi-nar-bundles/nifi-kite-bundle/nifi-kite-processors/src/main/java/org/apache/nifi/processors/kite/ConvertAvroSchema.java
@@ -36,6 +36,8 @@ import org.apache.avro.generic.GenericData.Record;
import org.apache.avro.generic.GenericDatumReader;
import org.apache.hadoop.conf.Configuration;
import org.apache.nifi.annotation.behavior.DynamicProperty;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.components.PropertyDescriptor;
@@ -62,6 +64,7 @@ import com.google.common.collect.Lists;
@Tags({ "avro", "convert", "kite" })
@CapabilityDescription("Convert records from one Avro schema to another, including support for flattening and simple type conversions")
+@InputRequirement(Requirement.INPUT_REQUIRED)
@DynamicProperty(name = "Field name from input schema",
value = "Field name for output schema",
description = "Explicit mappings from input schema to output schema, which supports renaming fields and stepping into nested records on the input schema using notation like parent.id")
http://git-wip-us.apache.org/repos/asf/nifi/blob/773576e0/nifi-nar-bundles/nifi-mongodb-bundle/nifi-mongodb-processors/src/main/java/org/apache/nifi/processors/mongodb/GetMongo.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-mongodb-bundle/nifi-mongodb-processors/src/main/java/org/apache/nifi/processors/mongodb/GetMongo.java b/nifi-nar-bundles/nifi-mongodb-bundle/nifi-mongodb-processors/src/main/java/org/apache/nifi/processors/mongodb/GetMongo.java
index 02ce9cf..9238db1 100644
--- a/nifi-nar-bundles/nifi-mongodb-bundle/nifi-mongodb-processors/src/main/java/org/apache/nifi/processors/mongodb/GetMongo.java
+++ b/nifi-nar-bundles/nifi-mongodb-bundle/nifi-mongodb-processors/src/main/java/org/apache/nifi/processors/mongodb/GetMongo.java
@@ -27,6 +27,8 @@ import java.util.List;
import java.util.Set;
import org.apache.commons.io.IOUtils;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.components.PropertyDescriptor;
@@ -48,6 +50,7 @@ import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoCursor;
@Tags({ "mongodb", "read", "get" })
+@InputRequirement(Requirement.INPUT_FORBIDDEN)
@CapabilityDescription("Creates FlowFiles from documents in MongoDB")
public class GetMongo extends AbstractMongoProcessor {
public static final Validator DOCUMENT_VALIDATOR = new Validator() {
http://git-wip-us.apache.org/repos/asf/nifi/blob/773576e0/nifi-nar-bundles/nifi-mongodb-bundle/nifi-mongodb-processors/src/main/java/org/apache/nifi/processors/mongodb/PutMongo.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-mongodb-bundle/nifi-mongodb-processors/src/main/java/org/apache/nifi/processors/mongodb/PutMongo.java b/nifi-nar-bundles/nifi-mongodb-bundle/nifi-mongodb-processors/src/main/java/org/apache/nifi/processors/mongodb/PutMongo.java
index 4b56f68..ae4009c 100644
--- a/nifi-nar-bundles/nifi-mongodb-bundle/nifi-mongodb-processors/src/main/java/org/apache/nifi/processors/mongodb/PutMongo.java
+++ b/nifi-nar-bundles/nifi-mongodb-bundle/nifi-mongodb-processors/src/main/java/org/apache/nifi/processors/mongodb/PutMongo.java
@@ -26,6 +26,8 @@ import java.util.List;
import java.util.Set;
import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.components.PropertyDescriptor;
@@ -46,6 +48,7 @@ import com.mongodb.client.model.UpdateOptions;
@EventDriven
@Tags({ "mongodb", "insert", "update", "write", "put" })
+@InputRequirement(Requirement.INPUT_REQUIRED)
@CapabilityDescription("Writes the contents of a FlowFile to MongoDB")
public class PutMongo extends AbstractMongoProcessor {
static final Relationship REL_SUCCESS = new Relationship.Builder().name("success")
http://git-wip-us.apache.org/repos/asf/nifi/blob/773576e0/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateRegularExpression.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateRegularExpression.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateRegularExpression.java
index 50f10f3..4123d72 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateRegularExpression.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateRegularExpression.java
@@ -29,6 +29,8 @@ import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.stream.io.StreamUtils;
@@ -54,6 +56,7 @@ import org.apache.nifi.annotation.documentation.SeeAlso;
@SideEffectFree
@SupportsBatching
@Tags({"deprecated"})
+@InputRequirement(Requirement.INPUT_REQUIRED)
@CapabilityDescription("WARNING: This has been deprecated and will be removed in 0.2.0. \n\n Use ExtractText instead.")
@SeeAlso(ExtractText.class)
@Deprecated
[33/50] [abbrv] nifi git commit: Fixing one-character typo in syslog
attribute name
Posted by mc...@apache.org.
Fixing one-character typo in syslog attribute name
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/f1f67f63
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/f1f67f63
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/f1f67f63
Branch: refs/heads/NIFI-655
Commit: f1f67f6395e44775c70355688d04a46499176757
Parents: 08d59e4
Author: Bryan Bende <bb...@apache.org>
Authored: Fri Nov 20 10:45:21 2015 -0500
Committer: Bryan Bende <bb...@apache.org>
Committed: Fri Nov 20 10:45:21 2015 -0500
----------------------------------------------------------------------
.../apache/nifi/processors/standard/AbstractSyslogProcessor.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/f1f67f63/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractSyslogProcessor.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractSyslogProcessor.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractSyslogProcessor.java
index fea01bd..70d3f22 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractSyslogProcessor.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractSyslogProcessor.java
@@ -66,7 +66,7 @@ public abstract class AbstractSyslogProcessor extends AbstractProcessor {
BODY("syslog.body"),
VALID("syslog.valid"),
PROTOCOL("syslog.protocol"),
- PORT("syslog.pprt");
+ PORT("syslog.port");
private String key;
[44/50] [abbrv] nifi git commit: NIFI-655: - Refactoring web security
to use Spring Security Java Configuration. - Introducing security in Web UI
in order to get JWT.
Posted by mc...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/jwt/JwtService.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/jwt/JwtService.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/jwt/JwtService.java
new file mode 100644
index 0000000..9635354
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/jwt/JwtService.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.security.jwt;
+
+import io.jsonwebtoken.Claims;
+import io.jsonwebtoken.ExpiredJwtException;
+import io.jsonwebtoken.Jws;
+import io.jsonwebtoken.JwsHeader;
+import io.jsonwebtoken.JwtException;
+import io.jsonwebtoken.Jwts;
+import io.jsonwebtoken.MalformedJwtException;
+import io.jsonwebtoken.SignatureAlgorithm;
+import io.jsonwebtoken.SignatureException;
+import io.jsonwebtoken.SigningKeyResolverAdapter;
+import io.jsonwebtoken.UnsupportedJwtException;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.admin.service.AdministrationException;
+import org.apache.nifi.admin.service.KeyService;
+import org.apache.nifi.web.security.token.LoginAuthenticationToken;
+import org.slf4j.LoggerFactory;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Calendar;
+import org.apache.nifi.key.Key;
+
+/**
+ *
+ */
+public class JwtService {
+ private static final org.slf4j.Logger logger = LoggerFactory.getLogger(JwtService.class);
+
+ private static final SignatureAlgorithm SIGNATURE_ALGORITHM = SignatureAlgorithm.HS256;
+ private static final String KEY_ID_CLAIM = "kid";
+ private static final String USERNAME_CLAIM = "preferred_username";
+
+ private final KeyService keyService;
+
+ public JwtService(final KeyService keyService) {
+ this.keyService = keyService;
+ }
+
+ public String getAuthenticationFromToken(final String base64EncodedToken) throws JwtException {
+ // The library representations of the JWT should be kept internal to this service.
+ try {
+ final Jws<Claims> jws = parseTokenFromBase64EncodedString(base64EncodedToken);
+
+ if (jws == null) {
+ throw new JwtException("Unable to parse token");
+ }
+
+ // Additional validation that subject is present
+ if (StringUtils.isEmpty(jws.getBody().getSubject())) {
+ throw new JwtException("No subject available in token");
+ }
+
+ // TODO: Validate issuer against active registry?
+ if (StringUtils.isEmpty(jws.getBody().getIssuer())) {
+ // TODO: Remove after testing
+// logger.info("Decoded JWT payload: " + jws.toString());
+ throw new JwtException("No issuer available in token");
+ }
+ return jws.getBody().getSubject();
+ } catch (JwtException e) {
+ logger.debug("The Base64 encoded JWT: " + base64EncodedToken);
+ final String errorMessage = "There was an error validating the JWT";
+ logger.error(errorMessage, e);
+ throw e;
+ }
+ }
+
+ private Jws<Claims> parseTokenFromBase64EncodedString(final String base64EncodedToken) throws JwtException {
+ try {
+ return Jwts.parser().setSigningKeyResolver(new SigningKeyResolverAdapter() {
+ @Override
+ public byte[] resolveSigningKeyBytes(JwsHeader header, Claims claims) {
+ final String identity = claims.getSubject();
+
+ // Get the key based on the key id in the claims
+ final Integer keyId = claims.get(KEY_ID_CLAIM, Integer.class);
+ final Key key = keyService.getKey(keyId);
+
+ // Ensure we were able to find a key that was previously issued by this key service for this user
+ if (key == null || key.getKey() == null) {
+ throw new UnsupportedJwtException("Unable to determine signing key for " + identity + " [kid: " + keyId + "]");
+ }
+
+ return key.getKey().getBytes(StandardCharsets.UTF_8);
+ }
+ }).parseClaimsJws(base64EncodedToken);
+ } catch (final MalformedJwtException | UnsupportedJwtException | SignatureException | ExpiredJwtException | IllegalArgumentException | AdministrationException e) {
+ // TODO: Exercise all exceptions to ensure none leak key material to logs
+ final String errorMessage = "There was an error validating the JWT";
+ throw new JwtException(errorMessage, e);
+ }
+ }
+
+ /**
+ * Generates a signed JWT token from the provided (Spring Security) login authentication token.
+ *
+ * @param authenticationToken an instance of the Spring Security token after login credentials have been verified against the respective information source
+ * @return a signed JWT containing the user identity and the identity provider, Base64-encoded
+ * @throws JwtException if there is a problem generating the signed token
+ */
+ public String generateSignedToken(final LoginAuthenticationToken authenticationToken) throws JwtException {
+ if (authenticationToken == null) {
+ throw new IllegalArgumentException("Cannot generate a JWT for a null authentication token");
+ }
+
+ // Set expiration from the token
+ final Calendar expiration = Calendar.getInstance();
+ expiration.setTimeInMillis(authenticationToken.getExpiration());
+
+ final Object principal = authenticationToken.getPrincipal();
+ if (principal == null || StringUtils.isEmpty(principal.toString())) {
+ final String errorMessage = "Cannot generate a JWT for a token with an empty identity issued by " + authenticationToken.getIssuer();
+ logger.error(errorMessage);
+ throw new JwtException(errorMessage);
+ }
+
+ // Create a JWT with the specified authentication
+ final String identity = principal.toString();
+ final String username = authenticationToken.getName();
+
+ try {
+ // Get/create the key for this user
+ final Key key = keyService.getOrCreateKey(identity);
+ final byte[] keyBytes = key.getKey().getBytes(StandardCharsets.UTF_8);
+
+ logger.trace("Generating JWT for " + authenticationToken);
+
+ // TODO: Implement "jti" claim with nonce to prevent replay attacks and allow blacklisting of revoked tokens
+
+ // Build the token
+ return Jwts.builder().setSubject(identity)
+ .setIssuer(authenticationToken.getIssuer())
+ .setAudience(authenticationToken.getIssuer())
+ .claim(USERNAME_CLAIM, username)
+ .claim(KEY_ID_CLAIM, key.getId())
+ .setExpiration(expiration.getTime())
+ .setIssuedAt(Calendar.getInstance().getTime())
+ .signWith(SIGNATURE_ALGORITHM, keyBytes).compact();
+ } catch (NullPointerException | AdministrationException e) {
+ final String errorMessage = "Could not retrieve the signing key for JWT for " + identity;
+ logger.error(errorMessage, e);
+ throw new JwtException(errorMessage, e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/node/NodeAuthorizedUserFilter.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/node/NodeAuthorizedUserFilter.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/node/NodeAuthorizedUserFilter.java
new file mode 100644
index 0000000..a3e6c3c
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/node/NodeAuthorizedUserFilter.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.security.node;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.security.cert.X509Certificate;
+import javax.servlet.FilterChain;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
+import org.apache.nifi.controller.FlowController;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.authentication.AuthenticationResponse;
+import org.apache.nifi.web.security.user.NiFiUserDetails;
+import org.apache.nifi.user.NiFiUser;
+import org.apache.nifi.util.NiFiProperties;
+import org.apache.nifi.web.security.token.NiFiAuthorizationToken;
+import org.apache.nifi.web.security.x509.X509CertificateExtractor;
+import org.apache.nifi.web.security.x509.X509IdentityProvider;
+import org.apache.nifi.web.util.WebUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.ApplicationContext;
+import org.springframework.security.core.context.SecurityContextHolder;
+import org.springframework.web.context.support.WebApplicationContextUtils;
+import org.springframework.web.filter.GenericFilterBean;
+
+/**
+ * Custom filter to extract a user's authorities from the request where the user was authenticated by the cluster manager and populate the threadlocal with the authorized user. If the request contains
+ * the appropriate header with authorities and the application instance is a node connected to the cluster, then the authentication/authorization steps remaining in the filter chain are skipped.
+ *
+ * Checking if the application instance is a connected node is important because it prevents external clients from faking the request headers and bypassing the authentication processing chain.
+ */
+public class NodeAuthorizedUserFilter extends GenericFilterBean {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(NodeAuthorizedUserFilter.class);
+
+ public static final String PROXY_USER_DETAILS = "X-ProxiedEntityUserDetails";
+
+ private NiFiProperties properties;
+ private X509CertificateExtractor certificateExtractor;
+ private X509IdentityProvider certificateIdentityProvider;
+
+ @Override
+ public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException {
+ final HttpServletRequest httpServletRequest = (HttpServletRequest) request;
+
+ // get the proxied user's authorities
+ final String hexEncodedUserDetails = httpServletRequest.getHeader(PROXY_USER_DETAILS);
+
+ // check if the request has the necessary header information and this instance is configured as a node
+ if (StringUtils.isNotBlank(hexEncodedUserDetails) && properties.isNode()) {
+
+ // get the flow controller from the Spring context
+ final ApplicationContext ctx = WebApplicationContextUtils.getWebApplicationContext(getServletContext());
+ final FlowController flowController = ctx.getBean("flowController", FlowController.class);
+
+ // check that we are connected to the cluster
+ if (flowController.getNodeId() != null) {
+ try {
+ // attempt to extract the client certificate
+ final X509Certificate[] certificate = certificateExtractor.extractClientCertificate(httpServletRequest);
+ if (certificate != null) {
+ // authenticate the certificate
+ final AuthenticationResponse authenticationResponse = certificateIdentityProvider.authenticate(certificate);
+
+ // only consider the pre-authorized user when the request came directly from the NCM according to the DN in the certificate
+ final String clusterManagerIdentity = flowController.getClusterManagerDN();
+ if (clusterManagerIdentity != null && clusterManagerIdentity.equals(authenticationResponse.getIdentity())) {
+ // deserialize hex encoded object
+ final Serializable userDetailsObj = WebUtils.deserializeHexToObject(hexEncodedUserDetails);
+
+ // if we have a valid object, set the authentication token and bypass the remaining authentication processing chain
+ if (userDetailsObj instanceof NiFiUserDetails) {
+ final NiFiUserDetails userDetails = (NiFiUserDetails) userDetailsObj;
+ final NiFiUser user = userDetails.getNiFiUser();
+
+ // log the request attempt - response details will be logged later
+ logger.info(String.format("Attempting request for (%s) %s %s (source ip: %s)", user.getIdentity(), httpServletRequest.getMethod(),
+ httpServletRequest.getRequestURL().toString(), request.getRemoteAddr()));
+
+ // create the authorized nifi token
+ final NiFiAuthorizationToken token = new NiFiAuthorizationToken(userDetails);
+ SecurityContextHolder.getContext().setAuthentication(token);
+ }
+ }
+ }
+ } catch (final ClassNotFoundException cnfe) {
+ LOGGER.warn("Classpath issue detected because failed to deserialize authorized user in request header due to: " + cnfe, cnfe);
+ } catch (final IllegalArgumentException iae) {
+ // unable to authenticate a serialized user from the incoming request
+ }
+ }
+ }
+
+ chain.doFilter(request, response);
+ }
+
+ public void setProperties(NiFiProperties properties) {
+ this.properties = properties;
+ }
+
+ public void setCertificateIdentityProvider(X509IdentityProvider certificateIdentityProvider) {
+ this.certificateIdentityProvider = certificateIdentityProvider;
+ }
+
+ public void setCertificateExtractor(X509CertificateExtractor certificateExtractor) {
+ this.certificateExtractor = certificateExtractor;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/spring/LoginIdentityProviderFactoryBean.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/spring/LoginIdentityProviderFactoryBean.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/spring/LoginIdentityProviderFactoryBean.java
new file mode 100644
index 0000000..92a27ae
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/spring/LoginIdentityProviderFactoryBean.java
@@ -0,0 +1,312 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.security.spring;
+
+import java.io.File;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.HashMap;
+import java.util.Map;
+import javax.xml.XMLConstants;
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBElement;
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.Unmarshaller;
+import javax.xml.transform.stream.StreamSource;
+import javax.xml.validation.Schema;
+import javax.xml.validation.SchemaFactory;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.authentication.AuthenticationResponse;
+import org.apache.nifi.authentication.LoginCredentials;
+import org.apache.nifi.authentication.LoginIdentityProvider;
+import org.apache.nifi.authentication.LoginIdentityProviderConfigurationContext;
+import org.apache.nifi.authentication.LoginIdentityProviderInitializationContext;
+import org.apache.nifi.authentication.LoginIdentityProviderLookup;
+import org.apache.nifi.authentication.annotation.LoginIdentityProviderContext;
+import org.apache.nifi.authentication.generated.LoginIdentityProviders;
+import org.apache.nifi.authentication.generated.Property;
+import org.apache.nifi.authentication.generated.Provider;
+import org.apache.nifi.authorization.exception.ProviderCreationException;
+import org.apache.nifi.authorization.exception.ProviderDestructionException;
+import org.apache.nifi.nar.ExtensionManager;
+import org.apache.nifi.nar.NarCloseable;
+import org.apache.nifi.util.NiFiProperties;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.DisposableBean;
+import org.springframework.beans.factory.FactoryBean;
+import org.xml.sax.SAXException;
+
+/**
+ *
+ */
+public class LoginIdentityProviderFactoryBean implements FactoryBean, DisposableBean, LoginIdentityProviderLookup {
+
+ private static final Logger logger = LoggerFactory.getLogger(LoginIdentityProviderFactoryBean.class);
+ private static final String LOGIN_IDENTITY_PROVIDERS_XSD = "/login-identity-providers.xsd";
+ private static final String JAXB_GENERATED_PATH = "org.apache.nifi.authentication.generated";
+ private static final JAXBContext JAXB_CONTEXT = initializeJaxbContext();
+
+ /**
+ * Load the JAXBContext.
+ */
+ private static JAXBContext initializeJaxbContext() {
+ try {
+ return JAXBContext.newInstance(JAXB_GENERATED_PATH, LoginIdentityProviderFactoryBean.class.getClassLoader());
+ } catch (JAXBException e) {
+ throw new RuntimeException("Unable to create JAXBContext.");
+ }
+ }
+
+ private NiFiProperties properties;
+ private LoginIdentityProvider loginIdentityProvider;
+ private final Map<String, LoginIdentityProvider> loginIdentityProviders = new HashMap<>();
+
+ @Override
+ public LoginIdentityProvider getLoginIdentityProvider(String identifier) {
+ return loginIdentityProviders.get(identifier);
+ }
+
+ @Override
+ public Object getObject() throws Exception {
+ if (loginIdentityProvider == null) {
+ // look up the login identity provider to use
+ final String loginIdentityProviderIdentifier = properties.getProperty(NiFiProperties.SECURITY_USER_LOGIN_IDENTITY_PROVIDER);
+
+ // ensure the login identity provider class name was specified
+ if (StringUtils.isNotBlank(loginIdentityProviderIdentifier)) {
+ final LoginIdentityProviders loginIdentityProviderConfiguration = loadLoginIdentityProvidersConfiguration();
+
+ // create each login identity provider
+ for (final Provider provider : loginIdentityProviderConfiguration.getProvider()) {
+ loginIdentityProviders.put(provider.getIdentifier(), createLoginIdentityProvider(provider.getIdentifier(), provider.getClazz()));
+ }
+
+ // configure each login identity provider
+ for (final Provider provider : loginIdentityProviderConfiguration.getProvider()) {
+ final LoginIdentityProvider instance = loginIdentityProviders.get(provider.getIdentifier());
+ instance.onConfigured(loadLoginIdentityProviderConfiguration(provider));
+ }
+
+ // get the login identity provider instance
+ loginIdentityProvider = getLoginIdentityProvider(loginIdentityProviderIdentifier);
+
+ // ensure it was found
+ if (loginIdentityProvider == null) {
+ throw new Exception(String.format("The specified login identity provider '%s' could not be found.", loginIdentityProviderIdentifier));
+ }
+ }
+ }
+
+ return loginIdentityProvider;
+ }
+
+ private LoginIdentityProviders loadLoginIdentityProvidersConfiguration() throws Exception {
+ final File loginIdentityProvidersConfigurationFile = properties.getLoginIdentityProviderConfiguraitonFile();
+
+ // load the users from the specified file
+ if (loginIdentityProvidersConfigurationFile.exists()) {
+ try {
+ // find the schema
+ final SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
+ final Schema schema = schemaFactory.newSchema(LoginIdentityProviders.class.getResource(LOGIN_IDENTITY_PROVIDERS_XSD));
+
+ // attempt to unmarshal
+ final Unmarshaller unmarshaller = JAXB_CONTEXT.createUnmarshaller();
+ unmarshaller.setSchema(schema);
+ final JAXBElement<LoginIdentityProviders> element = unmarshaller.unmarshal(new StreamSource(loginIdentityProvidersConfigurationFile), LoginIdentityProviders.class);
+ return element.getValue();
+ } catch (SAXException | JAXBException e) {
+ throw new Exception("Unable to load the login identity provider configuration file at: " + loginIdentityProvidersConfigurationFile.getAbsolutePath());
+ }
+ } else {
+ throw new Exception("Unable to find the login identity provider configuration file at " + loginIdentityProvidersConfigurationFile.getAbsolutePath());
+ }
+ }
+
+ private LoginIdentityProvider createLoginIdentityProvider(final String identifier, final String loginIdentityProviderClassName) throws Exception {
+ // get the classloader for the specified login identity provider
+ final ClassLoader loginIdentityProviderClassLoader = ExtensionManager.getClassLoader(loginIdentityProviderClassName);
+ if (loginIdentityProviderClassLoader == null) {
+ throw new Exception(String.format("The specified login identity provider class '%s' is not known to this nifi.", loginIdentityProviderClassName));
+ }
+
+ // get the current context classloader
+ final ClassLoader currentClassLoader = Thread.currentThread().getContextClassLoader();
+
+ final LoginIdentityProvider instance;
+ try {
+ // set the appropriate class loader
+ Thread.currentThread().setContextClassLoader(loginIdentityProviderClassLoader);
+
+ // attempt to load the class
+ Class<?> rawLoginIdentityProviderClass = Class.forName(loginIdentityProviderClassName, true, loginIdentityProviderClassLoader);
+ Class<? extends LoginIdentityProvider> loginIdentityProviderClass = rawLoginIdentityProviderClass.asSubclass(LoginIdentityProvider.class);
+
+ // otherwise create a new instance
+ Constructor constructor = loginIdentityProviderClass.getConstructor();
+ instance = (LoginIdentityProvider) constructor.newInstance();
+
+ // method injection
+ performMethodInjection(instance, loginIdentityProviderClass);
+
+ // field injection
+ performFieldInjection(instance, loginIdentityProviderClass);
+
+ // call post construction lifecycle event
+ instance.initialize(new StandardLoginIdentityProviderInitializationContext(identifier, this));
+ } finally {
+ if (currentClassLoader != null) {
+ Thread.currentThread().setContextClassLoader(currentClassLoader);
+ }
+ }
+
+ return withNarLoader(instance);
+ }
+
+ private LoginIdentityProviderConfigurationContext loadLoginIdentityProviderConfiguration(final Provider provider) {
+ final Map<String, String> providerProperties = new HashMap<>();
+
+ for (final Property property : provider.getProperty()) {
+ providerProperties.put(property.getName(), property.getValue());
+ }
+
+ return new StandardLoginIdentityProviderConfigurationContext(provider.getIdentifier(), providerProperties);
+ }
+
+ private void performMethodInjection(final LoginIdentityProvider instance, final Class loginIdentityProviderClass)
+ throws IllegalAccessException, IllegalArgumentException, InvocationTargetException {
+
+ for (final Method method : loginIdentityProviderClass.getMethods()) {
+ if (method.isAnnotationPresent(LoginIdentityProviderContext.class)) {
+ // make the method accessible
+ final boolean isAccessible = method.isAccessible();
+ method.setAccessible(true);
+
+ try {
+ final Class<?>[] argumentTypes = method.getParameterTypes();
+
+ // look for setters (single argument)
+ if (argumentTypes.length == 1) {
+ final Class<?> argumentType = argumentTypes[0];
+
+ // look for well known types
+ if (NiFiProperties.class.isAssignableFrom(argumentType)) {
+ // nifi properties injection
+ method.invoke(instance, properties);
+ }
+ }
+ } finally {
+ method.setAccessible(isAccessible);
+ }
+ }
+ }
+
+ final Class parentClass = loginIdentityProviderClass.getSuperclass();
+ if (parentClass != null && LoginIdentityProvider.class.isAssignableFrom(parentClass)) {
+ performMethodInjection(instance, parentClass);
+ }
+ }
+
+ private void performFieldInjection(final LoginIdentityProvider instance, final Class loginIdentityProviderClass) throws IllegalArgumentException, IllegalAccessException {
+ for (final Field field : loginIdentityProviderClass.getDeclaredFields()) {
+ if (field.isAnnotationPresent(LoginIdentityProviderContext.class)) {
+ // make the method accessible
+ final boolean isAccessible = field.isAccessible();
+ field.setAccessible(true);
+
+ try {
+ // get the type
+ final Class<?> fieldType = field.getType();
+
+ // only consider this field if it isn't set yet
+ if (field.get(instance) == null) {
+ // look for well known types
+ if (NiFiProperties.class.isAssignableFrom(fieldType)) {
+ // nifi properties injection
+ field.set(instance, properties);
+ }
+ }
+
+ } finally {
+ field.setAccessible(isAccessible);
+ }
+ }
+ }
+
+ final Class parentClass = loginIdentityProviderClass.getSuperclass();
+ if (parentClass != null && LoginIdentityProvider.class.isAssignableFrom(parentClass)) {
+ performFieldInjection(instance, parentClass);
+ }
+ }
+
+ private LoginIdentityProvider withNarLoader(final LoginIdentityProvider baseProvider) {
+ return new LoginIdentityProvider() {
+
+ @Override
+ public AuthenticationResponse authenticate(LoginCredentials credentials) {
+ try (final NarCloseable narCloseable = NarCloseable.withNarLoader()) {
+ return baseProvider.authenticate(credentials);
+ }
+ }
+
+ @Override
+ public void initialize(LoginIdentityProviderInitializationContext initializationContext) throws ProviderCreationException {
+ try (final NarCloseable narCloseable = NarCloseable.withNarLoader()) {
+ baseProvider.initialize(initializationContext);
+ }
+ }
+
+ @Override
+ public void onConfigured(LoginIdentityProviderConfigurationContext configurationContext) throws ProviderCreationException {
+ try (final NarCloseable narCloseable = NarCloseable.withNarLoader()) {
+ baseProvider.onConfigured(configurationContext);
+ }
+ }
+
+ @Override
+ public void preDestruction() throws ProviderDestructionException {
+ try (final NarCloseable narCloseable = NarCloseable.withNarLoader()) {
+ baseProvider.preDestruction();
+ }
+ }
+ };
+ }
+
+ @Override
+ public Class getObjectType() {
+ return LoginIdentityProvider.class;
+ }
+
+ @Override
+ public boolean isSingleton() {
+ return true;
+ }
+
+ @Override
+ public void destroy() throws Exception {
+ if (loginIdentityProvider != null) {
+ loginIdentityProvider.preDestruction();
+ }
+ }
+
+ public void setProperties(NiFiProperties properties) {
+ this.properties = properties;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/spring/StandardLoginIdentityProviderConfigurationContext.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/spring/StandardLoginIdentityProviderConfigurationContext.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/spring/StandardLoginIdentityProviderConfigurationContext.java
new file mode 100644
index 0000000..5c662c7
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/spring/StandardLoginIdentityProviderConfigurationContext.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.security.spring;
+
+import java.util.Collections;
+import java.util.Map;
+import org.apache.nifi.authentication.LoginIdentityProviderConfigurationContext;
+
+/**
+ *
+ */
+public class StandardLoginIdentityProviderConfigurationContext implements LoginIdentityProviderConfigurationContext {
+
+ private final String identifier;
+ private final Map<String, String> properties;
+
+ public StandardLoginIdentityProviderConfigurationContext(String identifier, Map<String, String> properties) {
+ this.identifier = identifier;
+ this.properties = properties;
+ }
+
+ @Override
+ public String getIdentifier() {
+ return identifier;
+ }
+
+ @Override
+ public Map<String, String> getProperties() {
+ return Collections.unmodifiableMap(properties);
+ }
+
+ @Override
+ public String getProperty(String property) {
+ return properties.get(property);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/spring/StandardLoginIdentityProviderInitializationContext.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/spring/StandardLoginIdentityProviderInitializationContext.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/spring/StandardLoginIdentityProviderInitializationContext.java
new file mode 100644
index 0000000..af54df9
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/spring/StandardLoginIdentityProviderInitializationContext.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.security.spring;
+
+import org.apache.nifi.authentication.LoginIdentityProviderInitializationContext;
+import org.apache.nifi.authentication.LoginIdentityProviderLookup;
+
+/**
+ *
+ */
+public class StandardLoginIdentityProviderInitializationContext implements LoginIdentityProviderInitializationContext {
+
+ private final String identifier;
+ private final LoginIdentityProviderLookup lookup;
+
+ public StandardLoginIdentityProviderInitializationContext(String identifier, final LoginIdentityProviderLookup lookup) {
+ this.identifier = identifier;
+ this.lookup = lookup;
+ }
+
+ @Override
+ public String getIdentifier() {
+ return identifier;
+ }
+
+ @Override
+ public LoginIdentityProviderLookup getAuthorityProviderLookup() {
+ return lookup;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/token/LoginAuthenticationToken.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/token/LoginAuthenticationToken.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/token/LoginAuthenticationToken.java
new file mode 100644
index 0000000..3591239
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/token/LoginAuthenticationToken.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.security.token;
+
+import org.apache.nifi.security.util.CertificateUtils;
+import org.springframework.security.authentication.AbstractAuthenticationToken;
+
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+
+/**
+ * This is an Authentication Token for logging in. Once a user is authenticated, they can be issued an ID token.
+ */
+public class LoginAuthenticationToken extends AbstractAuthenticationToken {
+
+ private final String identity;
+ private final String username;
+ private final long expiration;
+ private final String issuer;
+
+ /**
+ * Creates a representation of the authentication token for a user.
+ *
+ * @param identity The unique identifier for this user
+ * @param expiration The relative time to expiration in milliseconds
+ * @param issuer The IdentityProvider implementation that generated this token
+ */
+ public LoginAuthenticationToken(final String identity, final long expiration, final String issuer) {
+ this(identity, null, expiration, issuer);
+ }
+
+ /**
+ * Creates a representation of the authentication token for a user.
+ *
+ * @param identity The unique identifier for this user (cannot be null or empty)
+ * @param username The preferred username for this user
+ * @param expiration The relative time to expiration in milliseconds
+ * @param issuer The IdentityProvider implementation that generated this token
+ */
+ public LoginAuthenticationToken(final String identity, final String username, final long expiration, final String issuer) {
+ super(null);
+ setAuthenticated(true);
+ this.identity = identity;
+ this.username = username;
+ this.issuer = issuer;
+ Calendar now = Calendar.getInstance();
+ this.expiration = now.getTimeInMillis() + expiration;
+ }
+
+ @Override
+ public Object getCredentials() {
+ return null;
+ }
+
+ @Override
+ public Object getPrincipal() {
+ return identity;
+ }
+
+ /**
+ * Returns the expiration instant in milliseconds. This value is an absolute point in time (i.e. Nov
+ * 16, 2015 11:30:00.000 GMT), not a relative time (i.e. 60 minutes). It is calculated by adding the
+ * relative expiration from the constructor to the timestamp at object creation.
+ *
+ * @return the expiration in millis
+ */
+ public long getExpiration() {
+ return expiration;
+ }
+
+ public String getIssuer() {
+ return issuer;
+ }
+
+ @Override
+ public String getName() {
+ if (username == null) {
+ // if the username is a DN this will extract the username or CN... if not will return what was passed
+ return CertificateUtils.extractUsername(identity);
+ } else {
+ return username;
+ }
+ }
+
+ @Override
+ public String toString() {
+ Calendar expirationTime = Calendar.getInstance();
+ expirationTime.setTimeInMillis(getExpiration());
+ long remainingTime = expirationTime.getTimeInMillis() - Calendar.getInstance().getTimeInMillis();
+
+ SimpleDateFormat dateFormat = new SimpleDateFormat("dd-MM-yyyy HH:mm:ss.SSS");
+ dateFormat.setTimeZone(expirationTime.getTimeZone());
+ String expirationTimeString = dateFormat.format(expirationTime.getTime());
+
+ return new StringBuilder("LoginAuthenticationToken for ")
+ .append(getName())
+ .append(" issued by ")
+ .append(getIssuer())
+ .append(" expiring at ")
+ .append(expirationTimeString)
+ .append(" [")
+ .append(getExpiration())
+ .append(" ms, ")
+ .append(remainingTime)
+ .append(" ms remaining]")
+ .toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/token/NewAccountAuthenticationRequestToken.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/token/NewAccountAuthenticationRequestToken.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/token/NewAccountAuthenticationRequestToken.java
new file mode 100644
index 0000000..6fee4ec
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/token/NewAccountAuthenticationRequestToken.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.security.token;
+
+import org.apache.nifi.web.security.user.NewAccountRequest;
+
+/**
+ * This is an Authentication Token for a user that is requesting authentication in order to submit a new account request.
+ */
+public class NewAccountAuthenticationRequestToken extends NiFiAuthenticationRequestToken {
+
+ final NewAccountRequest newAccountRequest;
+
+ public NewAccountAuthenticationRequestToken(final NewAccountRequest newAccountRequest) {
+ super(newAccountRequest.getChain());
+ this.newAccountRequest = newAccountRequest;
+ }
+
+ public String getJustification() {
+ return newAccountRequest.getJustification();
+ }
+
+ public NewAccountRequest getNewAccountRequest() {
+ return newAccountRequest;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/token/NewAccountAuthenticationToken.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/token/NewAccountAuthenticationToken.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/token/NewAccountAuthenticationToken.java
new file mode 100644
index 0000000..5fe3a1d
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/token/NewAccountAuthenticationToken.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.security.token;
+
+import org.apache.nifi.web.security.user.NewAccountRequest;
+import org.springframework.security.authentication.AbstractAuthenticationToken;
+
+/**
+ * This is an Authentication Token for a user that has been authenticated but is not authorized to access the NiFi APIs. Typically, this authentication token is used successfully when requesting a
+ * NiFi account. Requesting any other endpoint would be rejected due to lack of roles.
+ */
+public class NewAccountAuthenticationToken extends AbstractAuthenticationToken {
+
+ final NewAccountRequest newAccountRequest;
+
+ public NewAccountAuthenticationToken(final NewAccountRequest newAccountRequest) {
+ super(null);
+ super.setAuthenticated(true);
+ this.newAccountRequest = newAccountRequest;
+ }
+
+ @Override
+ public Object getCredentials() {
+ return null;
+ }
+
+ @Override
+ public Object getPrincipal() {
+ return newAccountRequest;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/token/NiFiAuthenticationRequestToken.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/token/NiFiAuthenticationRequestToken.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/token/NiFiAuthenticationRequestToken.java
new file mode 100644
index 0000000..3ae6491
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/token/NiFiAuthenticationRequestToken.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.security.token;
+
+import java.util.Collections;
+import java.util.List;
+import org.springframework.security.authentication.AbstractAuthenticationToken;
+
+/**
+ * An authentication token that is used as an authentication request. The request chain is specified during creation and is used authenticate the user(s). If the user is authenticated, the token is
+ * used to authorized the user(s).
+ */
+public class NiFiAuthenticationRequestToken extends AbstractAuthenticationToken {
+
+ private final List<String> chain;
+
+ public NiFiAuthenticationRequestToken(final List<String> chain) {
+ super(null);
+ this.chain = chain;
+ }
+
+ @Override
+ public Object getCredentials() {
+ return null;
+ }
+
+ @Override
+ public Object getPrincipal() {
+ return chain;
+ }
+
+ public List<String> getChain() {
+ return Collections.unmodifiableList(chain);
+ }
+
+ @Override
+ public final void setAuthenticated(boolean authenticated) {
+ throw new IllegalArgumentException("Cannot change the authenticated state.");
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/token/NiFiAuthorizationToken.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/token/NiFiAuthorizationToken.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/token/NiFiAuthorizationToken.java
new file mode 100644
index 0000000..0cb0353
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/token/NiFiAuthorizationToken.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.security.token;
+
+import org.springframework.security.authentication.AbstractAuthenticationToken;
+import org.springframework.security.core.userdetails.UserDetails;
+
+/**
+ * An authentication token that represents an Authenticated and Authorized user of the NiFi Apis. The authorities are based off the specified UserDetails.
+ */
+public class NiFiAuthorizationToken extends AbstractAuthenticationToken {
+
+ final UserDetails nifiUserDetails;
+
+ public NiFiAuthorizationToken(final UserDetails nifiUserDetails) {
+ super(nifiUserDetails.getAuthorities());
+ super.setAuthenticated(true);
+ setDetails(nifiUserDetails);
+ this.nifiUserDetails = nifiUserDetails;
+ }
+
+ @Override
+ public Object getCredentials() {
+ return nifiUserDetails.getPassword();
+ }
+
+ @Override
+ public Object getPrincipal() {
+ return nifiUserDetails;
+ }
+
+ @Override
+ public final void setAuthenticated(boolean authenticated) {
+ throw new IllegalArgumentException("Cannot change the authenticated state.");
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/user/NewAccountRequest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/user/NewAccountRequest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/user/NewAccountRequest.java
new file mode 100644
index 0000000..3ec147a
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/user/NewAccountRequest.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.security.user;
+
+import java.util.List;
+
+/**
+ *
+ */
+public class NewAccountRequest {
+
+ private final List<String> chain;
+ private final String justification;
+
+ public NewAccountRequest(final List<String> chain, final String justification) {
+ this.chain = chain;
+ this.justification = justification;
+ }
+
+ public List<String> getChain() {
+ return chain;
+ }
+
+ public String getJustification() {
+ return justification;
+ }
+
+ public String getUsername() {
+ // the end user is the first item in the chain
+ return chain.get(0);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/user/NiFiUserDetails.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/user/NiFiUserDetails.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/user/NiFiUserDetails.java
index c69b1e6..b559269 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/user/NiFiUserDetails.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/user/NiFiUserDetails.java
@@ -73,10 +73,9 @@ public class NiFiUserDetails implements UserDetails {
@Override
public String getUsername() {
- return user.getDn();
+ return user.getIdentity();
}
- // TODO: not sure how to handle these yet
@Override
public boolean isAccountNonExpired() {
return true;
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/user/NiFiUserUtils.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/user/NiFiUserUtils.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/user/NiFiUserUtils.java
index bf1fe43..341663e 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/user/NiFiUserUtils.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/user/NiFiUserUtils.java
@@ -25,8 +25,7 @@ import org.springframework.security.core.context.SecurityContext;
import org.springframework.security.core.context.SecurityContextHolder;
/**
- * Utility methods for retrieving information about the current application
- * user.
+ * Utility methods for retrieving information about the current application user.
*
*/
public final class NiFiUserUtils {
@@ -58,8 +57,7 @@ public final class NiFiUserUtils {
}
/**
- * Returns the current NiFiUser or null if the current user is not a
- * NiFiUser.
+ * Returns the current NiFiUser or null if the current user is not a NiFiUser.
*
* @return user
*/
@@ -79,6 +77,27 @@ public final class NiFiUserUtils {
return user;
}
+ /**
+ * Returns the NewAccountRequest or null if this is not a new account request.
+ *
+ * @return new account request
+ */
+ public static NewAccountRequest getNewAccountRequest() {
+ NewAccountRequest newAccountRequest = null;
+
+ // obtain the principal in the current authentication
+ final SecurityContext context = SecurityContextHolder.getContext();
+ final Authentication authentication = context.getAuthentication();
+ if (authentication != null) {
+ Object principal = authentication.getPrincipal();
+ if (principal instanceof NewAccountRequest) {
+ newAccountRequest = (NewAccountRequest) principal;
+ }
+ }
+
+ return newAccountRequest;
+ }
+
public static String getNiFiUserName() {
// get the nifi user to extract the username
NiFiUser user = NiFiUserUtils.getNiFiUser();
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/x509/X509AuthenticationFilter.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/x509/X509AuthenticationFilter.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/x509/X509AuthenticationFilter.java
index 72baecb..dd7d47e 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/x509/X509AuthenticationFilter.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/x509/X509AuthenticationFilter.java
@@ -16,302 +16,66 @@
*/
package org.apache.nifi.web.security.x509;
-import org.apache.nifi.web.security.x509.ocsp.OcspCertificateValidator;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.security.cert.CertificateExpiredException;
-import java.security.cert.CertificateNotYetValidException;
import java.security.cert.X509Certificate;
-import javax.servlet.FilterChain;
-import javax.servlet.ServletException;
-import javax.servlet.ServletRequest;
-import javax.servlet.ServletResponse;
+import java.util.List;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-import org.apache.nifi.admin.service.AdministrationException;
-import org.apache.nifi.admin.service.UserService;
-import org.apache.nifi.web.security.DnUtils;
-import org.apache.nifi.web.security.UntrustedProxyException;
-import org.apache.nifi.util.NiFiProperties;
-import org.apache.commons.lang3.StringUtils;
-import org.springframework.security.authentication.AccountStatusException;
-import org.springframework.security.authentication.AuthenticationServiceException;
-import org.springframework.security.core.Authentication;
-import org.springframework.security.core.AuthenticationException;
-import org.springframework.security.core.userdetails.UsernameNotFoundException;
-import org.springframework.security.web.authentication.preauth.AbstractPreAuthenticatedProcessingFilter;
-import org.springframework.security.web.authentication.preauth.x509.X509PrincipalExtractor;
+import org.apache.nifi.authentication.AuthenticationResponse;
+import org.apache.nifi.web.security.InvalidAuthenticationException;
+import org.apache.nifi.web.security.NiFiAuthenticationFilter;
+import org.apache.nifi.web.security.ProxiedEntitiesUtils;
+import org.apache.nifi.web.security.token.NewAccountAuthenticationRequestToken;
+import org.apache.nifi.web.security.token.NiFiAuthenticationRequestToken;
+import org.apache.nifi.web.security.user.NewAccountRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
- * Custom X509 filter that will inspect the HTTP headers for a proxied user
- * before extracting the user details from the client certificate.
+ * Custom X509 filter that will inspect the HTTP headers for a proxied user before extracting the user details from the client certificate.
*/
-public class X509AuthenticationFilter extends AbstractPreAuthenticatedProcessingFilter {
+public class X509AuthenticationFilter extends NiFiAuthenticationFilter {
- public static final String PROXY_ENTITIES_CHAIN = "X-ProxiedEntitiesChain";
- public static final String PROXY_ENTITIES_ACCEPTED = "X-ProxiedEntitiesAccepted";
- public static final String PROXY_ENTITIES_DETAILS = "X-ProxiedEntitiesDetails";
+ private static final Logger logger = LoggerFactory.getLogger(X509AuthenticationFilter.class);
- private final X509CertificateExtractor certificateExtractor = new X509CertificateExtractor();
- private final X509PrincipalExtractor principalExtractor = new SubjectDnX509PrincipalExtractor();
- private OcspCertificateValidator certificateValidator;
- private NiFiProperties properties;
- private UserService userService;
+ private X509CertificateExtractor certificateExtractor;
+ private X509IdentityProvider certificateIdentityProvider;
@Override
- public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException {
- final HttpServletResponse httpResponse = (HttpServletResponse) response;
-
- // determine if this request is attempting to create a new account
- if (isNewAccountRequest((HttpServletRequest) request)) {
- // determine if this nifi supports new account requests
- if (properties.getSupportNewAccountRequests()) {
- // ensure there is a certificate in the request
- X509Certificate certificate = certificateExtractor.extractClientCertificate((HttpServletRequest) request);
- if (certificate != null) {
- // extract the principal from the certificate
- Object certificatePrincipal = principalExtractor.extractPrincipal(certificate);
- String principal = certificatePrincipal.toString();
-
- // log the new user account request
- logger.info("Requesting new user account for " + principal);
-
- try {
- // get the justification
- String justification = request.getParameter("justification");
- if (justification == null) {
- justification = StringUtils.EMPTY;
- }
-
- // create the pending user account
- userService.createPendingUserAccount(principal, justification);
-
- // generate a response
- httpResponse.setStatus(HttpServletResponse.SC_CREATED);
- httpResponse.setContentType("text/plain");
-
- // write the response message
- PrintWriter out = response.getWriter();
- out.println("Not authorized. User account created. Authorization pending.");
- } catch (IllegalArgumentException iae) {
- handleUserServiceError((HttpServletRequest) request, httpResponse, HttpServletResponse.SC_BAD_REQUEST, iae.getMessage());
- } catch (AdministrationException ae) {
- handleUserServiceError((HttpServletRequest) request, httpResponse, HttpServletResponse.SC_INTERNAL_SERVER_ERROR, ae.getMessage());
- }
- } else {
- // can this really happen?
- handleMissingCertificate((HttpServletRequest) request, httpResponse);
- }
- } else {
- handleUserServiceError((HttpServletRequest) request, httpResponse, HttpServletResponse.SC_NOT_FOUND, "This NiFi does not support new account requests.");
- }
- } else {
- try {
- // this not a request to create a user account - try to authorize
- super.doFilter(request, response, chain);
- } catch (AuthenticationException ae) {
- // continue the filter chain since anonymous access should be supported
- if (!properties.getNeedClientAuth()) {
- chain.doFilter(request, response);
- } else {
- // create an appropriate response for the given exception
- handleUnsuccessfulAuthentication((HttpServletRequest) request, httpResponse, ae);
- }
- }
- }
- }
-
- @Override
- protected Object getPreAuthenticatedPrincipal(HttpServletRequest request) {
- String principal;
-
- // extract the cert
- X509Certificate certificate = certificateExtractor.extractClientCertificate(request);
-
- // ensure the cert was found
- if (certificate == null) {
+ public NiFiAuthenticationRequestToken attemptAuthentication(HttpServletRequest request, HttpServletResponse response) {
+ // only suppport x509 login when running securely
+ if (!request.isSecure()) {
return null;
}
- // extract the principal
- Object certificatePrincipal = principalExtractor.extractPrincipal(certificate);
- principal = DnUtils.formatProxyDn(certificatePrincipal.toString());
-
- try {
- // ensure the cert is valid
- certificate.checkValidity();
- } catch (CertificateExpiredException cee) {
- final String message = String.format("Client certificate for (%s) is expired.", principal);
- logger.info(message, cee);
- if (logger.isDebugEnabled()) {
- logger.debug("", cee);
- }
- return null;
- } catch (CertificateNotYetValidException cnyve) {
- final String message = String.format("Client certificate for (%s) is not yet valid.", principal);
- logger.info(message, cnyve);
- if (logger.isDebugEnabled()) {
- logger.debug("", cnyve);
- }
+ // look for a client certificate
+ final X509Certificate[] certificates = certificateExtractor.extractClientCertificate(request);
+ if (certificates == null) {
return null;
}
- // validate the certificate in question
+ // attempt to authenticate if certificates were found
+ final AuthenticationResponse authenticationResponse;
try {
- certificateValidator.validate(request);
- } catch (final Exception e) {
- logger.info(e.getMessage());
- if (logger.isDebugEnabled()) {
- logger.debug("", e);
- }
- return null;
+ authenticationResponse = certificateIdentityProvider.authenticate(certificates);
+ } catch (final IllegalArgumentException iae) {
+ throw new InvalidAuthenticationException(iae.getMessage(), iae);
}
- // look for a proxied user
- if (StringUtils.isNotBlank(request.getHeader(PROXY_ENTITIES_CHAIN))) {
- principal = request.getHeader(PROXY_ENTITIES_CHAIN) + principal;
- }
-
- // log the request attempt - response details will be logged later
- logger.info(String.format("Attempting request for (%s) %s %s (source ip: %s)", principal, request.getMethod(),
- request.getRequestURL().toString(), request.getRemoteAddr()));
-
- return principal;
- }
-
- @Override
- protected Object getPreAuthenticatedCredentials(HttpServletRequest request) {
- return certificateExtractor.extractClientCertificate(request);
- }
-
- @Override
- protected void successfulAuthentication(HttpServletRequest request, HttpServletResponse response, Authentication authResult) {
- if (StringUtils.isNotBlank(request.getHeader(PROXY_ENTITIES_CHAIN))) {
- response.setHeader(PROXY_ENTITIES_ACCEPTED, Boolean.TRUE.toString());
- }
- super.successfulAuthentication(request, response, authResult);
- }
-
- @Override
- protected void unsuccessfulAuthentication(HttpServletRequest request, HttpServletResponse response, AuthenticationException failed) {
- if (StringUtils.isNotBlank(request.getHeader(PROXY_ENTITIES_CHAIN))) {
- response.setHeader(PROXY_ENTITIES_DETAILS, failed.getMessage());
- }
- super.unsuccessfulAuthentication(request, response, failed);
- }
-
- /**
- * Determines if the specified request is attempting to register a new user
- * account.
- *
- * @param request http request
- * @return true if new user
- */
- private boolean isNewAccountRequest(HttpServletRequest request) {
- if ("POST".equalsIgnoreCase(request.getMethod())) {
- String path = request.getPathInfo();
- if (StringUtils.isNotBlank(path)) {
- if ("/controller/users".equals(path)) {
- return true;
- }
- }
- }
- return false;
- }
-
- /**
- * Handles requests that were unable to be authorized.
- *
- * @param request request
- * @param response response
- * @param ae ex
- * @throws IOException ex
- */
- private void handleUnsuccessfulAuthentication(HttpServletRequest request, HttpServletResponse response, AuthenticationException ae) throws IOException {
- // set the response status
- response.setContentType("text/plain");
-
- // write the response message
- PrintWriter out = response.getWriter();
-
- // use the type of authentication exception to determine the response code
- if (ae instanceof UsernameNotFoundException) {
- if (properties.getSupportNewAccountRequests()) {
- response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
- out.println("Not authorized.");
- } else {
- response.setStatus(HttpServletResponse.SC_FORBIDDEN);
- out.println("Access is denied.");
- }
- } else if (ae instanceof AccountStatusException) {
- response.setStatus(HttpServletResponse.SC_FORBIDDEN);
- out.println(ae.getMessage());
- } else if (ae instanceof UntrustedProxyException) {
- response.setStatus(HttpServletResponse.SC_FORBIDDEN);
- out.println(ae.getMessage());
- } else if (ae instanceof AuthenticationServiceException) {
- logger.error(String.format("Unable to authorize: %s", ae.getMessage()), ae);
- response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
- out.println(String.format("Unable to authorize: %s", ae.getMessage()));
+ final List<String> proxyChain = ProxiedEntitiesUtils.buildProxiedEntitiesChain(request, authenticationResponse.getIdentity());
+ if (isNewAccountRequest(request)) {
+ return new NewAccountAuthenticationRequestToken(new NewAccountRequest(proxyChain, getJustification(request)));
} else {
- logger.error(String.format("Unable to authorize: %s", ae.getMessage()), ae);
- response.setStatus(HttpServletResponse.SC_FORBIDDEN);
- out.println("Access is denied.");
+ return new NiFiAuthenticationRequestToken(proxyChain);
}
-
- // log the failure
- logger.info(String.format("Rejecting access to web api: %s", ae.getMessage()));
-
- // optionally log the stack trace
- if (logger.isDebugEnabled()) {
- logger.debug(StringUtils.EMPTY, ae);
- }
- }
-
- private void handleUserServiceError(HttpServletRequest request, HttpServletResponse response, int responseCode, String message) throws IOException {
- // set the response status
- response.setContentType("text/plain");
- response.setStatus(responseCode);
-
- // write the response message
- PrintWriter out = response.getWriter();
- out.println(message);
-
- // log the failure
- logger.info(String.format("Unable to process request because %s", message));
- }
-
- /**
- * Handles requests that failed because they were bad input.
- *
- * @param request request
- * @param response response
- * @throws IOException ioe
- */
- private void handleMissingCertificate(HttpServletRequest request, HttpServletResponse response) throws IOException {
- // set the response status
- response.setContentType("text/plain");
- response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
-
- // write the response message
- PrintWriter out = response.getWriter();
- out.println("Unable to process request because the user certificate was not specified.");
-
- // log the failure
- logger.info("Unable to process request because the user certificate was not specified.");
}
/* setters */
- public void setProperties(NiFiProperties properties) {
- this.properties = properties;
- }
-
- public void setUserService(UserService userService) {
- this.userService = userService;
+ public void setCertificateExtractor(X509CertificateExtractor certificateExtractor) {
+ this.certificateExtractor = certificateExtractor;
}
- public void setCertificateValidator(OcspCertificateValidator certificateValidator) {
- this.certificateValidator = certificateValidator;
+ public void setCertificateIdentityProvider(X509IdentityProvider certificateIdentityProvider) {
+ this.certificateIdentityProvider = certificateIdentityProvider;
}
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/x509/X509CertificateExtractor.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/x509/X509CertificateExtractor.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/x509/X509CertificateExtractor.java
index b40d5a5..98d0154 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/x509/X509CertificateExtractor.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/x509/X509CertificateExtractor.java
@@ -35,11 +35,11 @@ public class X509CertificateExtractor {
* @param request http request
* @return cert
*/
- public X509Certificate extractClientCertificate(HttpServletRequest request) {
+ public X509Certificate[] extractClientCertificate(HttpServletRequest request) {
X509Certificate[] certs = (X509Certificate[]) request.getAttribute("javax.servlet.request.X509Certificate");
if (certs != null && certs.length > 0) {
- return certs[0];
+ return certs;
}
if (logger.isDebugEnabled()) {
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/x509/X509CertificateValidator.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/x509/X509CertificateValidator.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/x509/X509CertificateValidator.java
new file mode 100644
index 0000000..cb56a11
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/x509/X509CertificateValidator.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.security.x509;
+
+import java.security.cert.CertificateExpiredException;
+import java.security.cert.CertificateNotYetValidException;
+import java.security.cert.X509Certificate;
+import org.apache.nifi.web.security.x509.ocsp.CertificateStatusException;
+import org.apache.nifi.web.security.x509.ocsp.OcspCertificateValidator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Extracts client certificates from Http requests.
+ */
+public class X509CertificateValidator {
+
+ private final Logger logger = LoggerFactory.getLogger(getClass());
+
+ private OcspCertificateValidator ocspValidator;
+
+ /**
+ * Extract the client certificate from the specified HttpServletRequest or null if none is specified.
+ *
+ * @param certificates the client certificates
+ * @throws java.security.cert.CertificateExpiredException cert is expired
+ * @throws java.security.cert.CertificateNotYetValidException cert is not yet valid
+ * @throws org.apache.nifi.web.security.x509.ocsp.CertificateStatusException ocsp validation issue
+ */
+ public void validateClientCertificate(final X509Certificate[] certificates)
+ throws CertificateExpiredException, CertificateNotYetValidException, CertificateStatusException {
+
+ // ensure the cert is valid
+ certificates[0].checkValidity();
+
+ // perform ocsp validator if necessary
+ ocspValidator.validate(certificates);
+ }
+
+ public void setOcspValidator(OcspCertificateValidator ocspValidator) {
+ this.ocspValidator = ocspValidator;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/x509/X509IdentityProvider.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/x509/X509IdentityProvider.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/x509/X509IdentityProvider.java
new file mode 100644
index 0000000..db0b529
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/x509/X509IdentityProvider.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.security.x509;
+
+import java.security.cert.CertificateExpiredException;
+import java.security.cert.CertificateNotYetValidException;
+import java.security.cert.X509Certificate;
+import java.util.concurrent.TimeUnit;
+import org.apache.nifi.authentication.AuthenticationResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.security.web.authentication.preauth.x509.X509PrincipalExtractor;
+
+/**
+ * Identity provider for extract the authenticating a ServletRequest with a X509Certificate.
+ */
+public class X509IdentityProvider {
+
+ private static final Logger logger = LoggerFactory.getLogger(X509IdentityProvider.class);
+
+ private final String issuer = getClass().getSimpleName();
+
+ private X509CertificateValidator certificateValidator;
+ private X509PrincipalExtractor principalExtractor;
+
+ /**
+ * Authenticates the specified request by checking certificate validity.
+ *
+ * @param certificates the client certificates
+ * @return an authentication response
+ * @throws IllegalArgumentException the request did not contain a valid certificate (or no certificate)
+ */
+ public AuthenticationResponse authenticate(final X509Certificate[] certificates) throws IllegalArgumentException {
+ // ensure the cert was found
+ if (certificates == null || certificates.length == 0) {
+ throw new IllegalArgumentException("The specified request does not contain a client certificate.");
+ }
+
+ // extract the principal
+ final Object certificatePrincipal = principalExtractor.extractPrincipal(certificates[0]);
+ final String principal = certificatePrincipal.toString();
+
+ try {
+ certificateValidator.validateClientCertificate(certificates);
+ } catch (CertificateExpiredException cee) {
+ final String message = String.format("Client certificate for (%s) is expired.", principal);
+ logger.info(message, cee);
+ if (logger.isDebugEnabled()) {
+ logger.debug("", cee);
+ }
+ throw new IllegalArgumentException(message, cee);
+ } catch (CertificateNotYetValidException cnyve) {
+ final String message = String.format("Client certificate for (%s) is not yet valid.", principal);
+ logger.info(message, cnyve);
+ if (logger.isDebugEnabled()) {
+ logger.debug("", cnyve);
+ }
+ throw new IllegalArgumentException(message, cnyve);
+ } catch (final Exception e) {
+ logger.info(e.getMessage());
+ if (logger.isDebugEnabled()) {
+ logger.debug("", e);
+ }
+ throw new IllegalArgumentException(e.getMessage(), e);
+ }
+
+ // build the authentication response
+ return new AuthenticationResponse(principal, principal, TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS), issuer);
+ }
+
+ /* setters */
+ public void setCertificateValidator(X509CertificateValidator certificateValidator) {
+ this.certificateValidator = certificateValidator;
+ }
+
+ public void setPrincipalExtractor(X509PrincipalExtractor principalExtractor) {
+ this.principalExtractor = principalExtractor;
+ }
+
+}
[27/50] [abbrv] nifi git commit: NIFI-1165: Fix for tests
TestRouteText and PutHDFS which did not succeed on Windows
Posted by mc...@apache.org.
NIFI-1165: Fix for tests TestRouteText and PutHDFS which did not succeed on Windows
Reviewed by Tony Kurc (tkurc@apache.org)
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/fb335ea2
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/fb335ea2
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/fb335ea2
Branch: refs/heads/NIFI-655
Commit: fb335ea282961a236381f78fe18939c15afb66a0
Parents: e862f7f
Author: Joe Percivall <jo...@yahoo.com>
Authored: Tue Nov 17 11:41:49 2015 -0500
Committer: Tony Kurc <tr...@gmail.com>
Committed: Thu Nov 19 01:01:34 2015 -0500
----------------------------------------------------------------------
.../nifi/processors/hadoop/PutHDFSTest.java | 24 ++++++++++++++++++++
.../nifi/processors/standard/TestRouteText.java | 15 +++++++++++-
2 files changed, 38 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/fb335ea2/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/PutHDFSTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/PutHDFSTest.java b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/PutHDFSTest.java
index d159336..fe3b5b8 100644
--- a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/PutHDFSTest.java
+++ b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/PutHDFSTest.java
@@ -19,6 +19,7 @@ package org.apache.nifi.processors.hadoop;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
import java.io.File;
import java.io.FileInputStream;
@@ -44,10 +45,23 @@ import org.apache.nifi.util.MockProcessContext;
import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
import org.junit.Assert;
+import org.junit.BeforeClass;
import org.junit.Test;
public class PutHDFSTest {
+ @BeforeClass
+ public static void setUp() throws Exception{
+ /*
+ * Running Hadoop on Windows requires a special build which will produce required binaries and native modules [1]. Since functionality
+ * provided by this module and validated by these test does not have any native implication we do not distribute required binaries and native modules
+ * to support running these tests in Windows environment, therefore they are ignored. You can also get more info from this StackOverflow thread [2]
+ *
+ * [1] https://wiki.apache.org/hadoop/Hadoop2OnWindows
+ * [2] http://stackoverflow.com/questions/19620642/failed-to-locate-the-winutils-binary-in-the-hadoop-binary-path
+ */
+ }
+
@Test
public void testValidators() {
TestRunner runner = TestRunners.newTestRunner(PutHDFS.class);
@@ -159,6 +173,9 @@ public class PutHDFSTest {
@Test
public void testPutFile() throws IOException {
+ // Refer to comment in the BeforeClass method for an explanation
+ assumeTrue(isNotWindows());
+
TestRunner runner = TestRunners.newTestRunner(PutHDFS.class);
runner.setProperty(PutHDFS.DIRECTORY, "target/test-classes");
runner.setProperty(PutHDFS.CONFLICT_RESOLUTION, "replace");
@@ -182,6 +199,9 @@ public class PutHDFSTest {
@Test
public void testPutFileWithException() throws IOException {
+ // Refer to comment in the BeforeClass method for an explanation
+ assumeTrue(isNotWindows());
+
String dirName = "target/testPutFileWrongPermissions";
File file = new File(dirName);
file.mkdirs();
@@ -213,4 +233,8 @@ public class PutHDFSTest {
fs.setPermission(p, new FsPermission(FsAction.EXECUTE, FsAction.EXECUTE, FsAction.EXECUTE));
fs.delete(p, true);
}
+
+ private boolean isNotWindows() {
+ return !System.getProperty("os.name").startsWith("Windows");
+ }
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/fb335ea2/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestRouteText.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestRouteText.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestRouteText.java
index 0cd0034..760986d 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestRouteText.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestRouteText.java
@@ -707,9 +707,22 @@ public class TestRouteText {
if (str == null || str.isEmpty()) {
return 0;
}
+
+ String lineSeparator;
+
+ if(str.contains("\r\n")){
+ lineSeparator = "\r\n";
+ } else if(str.contains("\n")){
+ lineSeparator = "\n";
+ } else if(str.contains("\r")){
+ lineSeparator = "\r";
+ } else {
+ return 1;
+ }
+
int lines = 0;
int pos = 0;
- while ((pos = str.indexOf(System.lineSeparator(), pos) + 1) != 0) {
+ while ((pos = str.indexOf(lineSeparator, pos) + 1) != 0) {
lines++;
}
return lines;
[40/50] [abbrv] nifi git commit: NIFI-655: - Refactoring web security
to use Spring Security Java Configuration. - Introducing security in Web UI
in order to get JWT.
Posted by mc...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/nf-storage.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/nf-storage.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/nf-storage.js
new file mode 100644
index 0000000..7b0c3f6
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/nf-storage.js
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* global nf, d3 */
+
+nf.Storage = (function () {
+
+ // Store items for two days before being eligible for removal.
+ var TWO_DAYS = nf.Common.MILLIS_PER_DAY * 2;
+
+ /**
+ * Checks the expiration for the specified entry.
+ *
+ * @param {object} entry
+ * @returns {boolean}
+ */
+ var checkExpiration = function (entry) {
+ if (nf.Common.isDefinedAndNotNull(entry.expires)) {
+ // get the expiration
+ var expires = new Date(entry.expires);
+ var now = new Date();
+
+ // return whether the expiration date has passed
+ return expires.valueOf() < now.valueOf();
+ } else {
+ return false;
+ }
+ };
+
+ /**
+ * Gets an enty for the key. The entry expiration is not checked.
+ *
+ * @param {string} key
+ */
+ var getEntry = function (key) {
+ try {
+ // parse the entry
+ var entry = JSON.parse(localStorage.getItem(key));
+
+ // ensure the entry and item are present
+ if (nf.Common.isDefinedAndNotNull(entry)) {
+ return entry;
+ } else {
+ return null;
+ }
+ } catch (e) {
+ return null;
+ }
+ };
+
+ return {
+ /**
+ * Initializes the storage. Items will be persisted for two days. Once the scripts runs
+ * thereafter, all eligible items will be removed. This strategy does not support persistence.
+ */
+ init: function () {
+ for (var i = 0; i < localStorage.length; i++) {
+ try {
+ // get the next item
+ var key = localStorage.key(i);
+
+ // attempt to get the item which will expire if necessary
+ nf.Storage.getItem(key);
+ } catch (e) {
+ }
+ }
+ },
+
+ /**
+ * Stores the specified item.
+ *
+ * @param {string} key
+ * @param {object} item
+ * @param {integer} expires
+ */
+ setItem: function (key, item, expires) {
+ // calculate the expiration
+ expires = nf.Common.isDefinedAndNotNull(expires) ? expires : new Date().valueOf() + TWO_DAYS;
+
+ // create the entry
+ var entry = {
+ expires: expires,
+ item: item
+ };
+
+ // store the item
+ localStorage.setItem(key, JSON.stringify(entry));
+ },
+
+ /**
+ * Returns whether there is an entry for this key. This will not check the expiration. If
+ * the entry is expired, it will return null on a subsequent getItem invocation.
+ *
+ * @param {string} key
+ * @returns {boolean}
+ */
+ hasItem: function (key) {
+ return getEntry(key) !== null;
+ },
+
+ /**
+ * Gets the item with the specified key. If an item with this key does
+ * not exist, null is returned. If an item exists but cannot be parsed
+ * or is malformed/unrecognized, null is returned.
+ *
+ * @param {type} key
+ */
+ getItem: function (key) {
+ var entry = getEntry(key);
+ if (entry === null) {
+ return null;
+ }
+
+ // if the entry is expired, drop it and return null
+ if (checkExpiration(entry)) {
+ nf.Storage.removeItem(key);
+ return null;
+ }
+
+ // if the entry has the specified field return its value
+ if (nf.Common.isDefinedAndNotNull(entry['item'])) {
+ return entry['item'];
+ } else {
+ return null;
+ }
+ },
+
+ /**
+ * Gets the expiration for the specified item. This will not check the expiration. If
+ * the entry is expired, it will return null on a subsequent getItem invocation.
+ *
+ * @param {string} key
+ * @returns {integer}
+ */
+ getItemExpiration: function (key) {
+ var entry = getEntry(key);
+ if (entry === null) {
+ return null;
+ }
+
+ // if the entry has the specified field return its value
+ if (nf.Common.isDefinedAndNotNull(entry['expires'])) {
+ return entry['expires'];
+ } else {
+ return null;
+ }
+ },
+
+ /**
+ * Removes the item with the specified key.
+ *
+ * @param {type} key
+ */
+ removeItem: function (key) {
+ localStorage.removeItem(key);
+ }
+ };
+}());
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/provenance/nf-provenance.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/provenance/nf-provenance.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/provenance/nf-provenance.js
index 0edbe08..876e06d 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/provenance/nf-provenance.js
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/provenance/nf-provenance.js
@@ -174,6 +174,8 @@ nf.Provenance = (function () {
* Initializes the status page.
*/
init: function () {
+ nf.Storage.init();
+
// load the users authorities and detect if the NiFi is clustered
$.when(loadControllerConfig(), loadAuthorities(), detectedCluster()).done(function () {
// create the provenance table
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/summary/nf-summary.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/summary/nf-summary.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/summary/nf-summary.js
index 97f8626..4f06662 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/summary/nf-summary.js
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/summary/nf-summary.js
@@ -128,6 +128,8 @@ nf.Summary = (function () {
* Initializes the status page.
*/
init: function () {
+ nf.Storage.init();
+
// intialize the summary table
initializeSummaryTable().done(function () {
// load the table
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/templates/nf-templates.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/templates/nf-templates.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/templates/nf-templates.js
index b55bee2..678e352 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/templates/nf-templates.js
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/templates/nf-templates.js
@@ -204,6 +204,8 @@ nf.Templates = (function () {
* Initializes the templates page.
*/
init: function () {
+ nf.Storage.init();
+
// load the users authorities
loadAuthorities().done(function () {
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/users/nf-users.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/users/nf-users.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/users/nf-users.js
index 96f73a5..9364aec 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/users/nf-users.js
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/users/nf-users.js
@@ -116,6 +116,8 @@ nf.Users = (function () {
* Initializes the counters page.
*/
init: function () {
+ nf.Storage.init();
+
// load the users authorities
loadAuthorities().done(function () {
// create the counters table
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-ldap-iaa-providers-bundle/nifi-ldap-iaa-providers-nar/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-ldap-iaa-providers-bundle/nifi-ldap-iaa-providers-nar/pom.xml b/nifi-nar-bundles/nifi-ldap-iaa-providers-bundle/nifi-ldap-iaa-providers-nar/pom.xml
new file mode 100644
index 0000000..59681b9
--- /dev/null
+++ b/nifi-nar-bundles/nifi-ldap-iaa-providers-bundle/nifi-ldap-iaa-providers-nar/pom.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-ldap-iaa-providers-bundle</artifactId>
+ <version>0.3.1-SNAPSHOT</version>
+ </parent>
+ <artifactId>nifi-ldap-iaa-providers-nar</artifactId>
+ <packaging>nar</packaging>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-ldap-iaa-providers</artifactId>
+ </dependency>
+ </dependencies>
+ <name>nifi-ldap-iaa-providers-nar</name>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-ldap-iaa-providers-bundle/nifi-ldap-iaa-providers/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-ldap-iaa-providers-bundle/nifi-ldap-iaa-providers/pom.xml b/nifi-nar-bundles/nifi-ldap-iaa-providers-bundle/nifi-ldap-iaa-providers/pom.xml
new file mode 100644
index 0000000..5cbd6f9
--- /dev/null
+++ b/nifi-nar-bundles/nifi-ldap-iaa-providers-bundle/nifi-ldap-iaa-providers/pom.xml
@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-ldap-iaa-providers-bundle</artifactId>
+ <version>0.3.1-SNAPSHOT</version>
+ </parent>
+ <artifactId>nifi-ldap-iaa-providers</artifactId>
+ <packaging>jar</packaging>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-utils</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-security-utils</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-ldap</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-beans</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-context</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-tx</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
+ </dependencies>
+ <name>nifi-ldap-iaa-providers</name>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-ldap-iaa-providers-bundle/nifi-ldap-iaa-providers/src/main/java/org/apache/nifi/ldap/LdapAuthenticationStrategy.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-ldap-iaa-providers-bundle/nifi-ldap-iaa-providers/src/main/java/org/apache/nifi/ldap/LdapAuthenticationStrategy.java b/nifi-nar-bundles/nifi-ldap-iaa-providers-bundle/nifi-ldap-iaa-providers/src/main/java/org/apache/nifi/ldap/LdapAuthenticationStrategy.java
new file mode 100644
index 0000000..7124ce1
--- /dev/null
+++ b/nifi-nar-bundles/nifi-ldap-iaa-providers-bundle/nifi-ldap-iaa-providers/src/main/java/org/apache/nifi/ldap/LdapAuthenticationStrategy.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.ldap;
+
+/**
+ *
+ */
+public enum LdapAuthenticationStrategy {
+
+ ANONYMOUS,
+ SIMPLE,
+ START_TLS
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-ldap-iaa-providers-bundle/nifi-ldap-iaa-providers/src/main/java/org/apache/nifi/ldap/LdapProvider.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-ldap-iaa-providers-bundle/nifi-ldap-iaa-providers/src/main/java/org/apache/nifi/ldap/LdapProvider.java b/nifi-nar-bundles/nifi-ldap-iaa-providers-bundle/nifi-ldap-iaa-providers/src/main/java/org/apache/nifi/ldap/LdapProvider.java
new file mode 100644
index 0000000..4dc7ea4
--- /dev/null
+++ b/nifi-nar-bundles/nifi-ldap-iaa-providers-bundle/nifi-ldap-iaa-providers/src/main/java/org/apache/nifi/ldap/LdapProvider.java
@@ -0,0 +1,279 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.ldap;
+
+import java.io.IOException;
+import java.security.KeyManagementException;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.UnrecoverableKeyException;
+import java.security.cert.CertificateException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import javax.net.ssl.SSLContext;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.authentication.AuthenticationResponse;
+import org.apache.nifi.authentication.LoginCredentials;
+import org.apache.nifi.authentication.LoginIdentityProvider;
+import org.apache.nifi.authentication.LoginIdentityProviderConfigurationContext;
+import org.apache.nifi.authentication.LoginIdentityProviderInitializationContext;
+import org.apache.nifi.authentication.exception.IdentityAccessException;
+import org.apache.nifi.authentication.exception.InvalidLoginCredentialsException;
+import org.apache.nifi.authorization.exception.ProviderCreationException;
+import org.apache.nifi.authorization.exception.ProviderDestructionException;
+import org.apache.nifi.security.util.SslContextFactory;
+import org.apache.nifi.security.util.SslContextFactory.ClientAuth;
+import org.apache.nifi.util.FormatUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.ldap.CommunicationException;
+import org.springframework.ldap.core.support.AbstractTlsDirContextAuthenticationStrategy;
+import org.springframework.ldap.core.support.DefaultTlsDirContextAuthenticationStrategy;
+import org.springframework.ldap.core.support.LdapContextSource;
+import org.springframework.ldap.core.support.SimpleDirContextAuthenticationStrategy;
+import org.springframework.security.authentication.AuthenticationServiceException;
+import org.springframework.security.authentication.BadCredentialsException;
+import org.springframework.security.authentication.UsernamePasswordAuthenticationToken;
+import org.springframework.security.core.Authentication;
+import org.springframework.security.ldap.authentication.AbstractLdapAuthenticationProvider;
+import org.springframework.security.ldap.authentication.BindAuthenticator;
+import org.springframework.security.ldap.authentication.LdapAuthenticationProvider;
+import org.springframework.security.ldap.search.FilterBasedLdapUserSearch;
+import org.springframework.security.ldap.search.LdapUserSearch;
+import org.springframework.security.ldap.userdetails.LdapUserDetails;
+
+/**
+ * Abstract LDAP based implementation of a login identity provider.
+ */
+public class LdapProvider implements LoginIdentityProvider {
+
+ private static final Logger logger = LoggerFactory.getLogger(LdapProvider.class);
+
+ private AbstractLdapAuthenticationProvider provider;
+ private String issuer;
+ private long expiration;
+
+ @Override
+ public final void initialize(final LoginIdentityProviderInitializationContext initializationContext) throws ProviderCreationException {
+ this.issuer = getClass().getSimpleName();
+ }
+
+ @Override
+ public final void onConfigured(final LoginIdentityProviderConfigurationContext configurationContext) throws ProviderCreationException {
+ final String rawExpiration = configurationContext.getProperty("Authentication Expiration");
+ if (StringUtils.isBlank(rawExpiration)) {
+ throw new ProviderCreationException("The Authentication Expiration must be specified.");
+ }
+
+ try {
+ expiration = FormatUtils.getTimeDuration(rawExpiration, TimeUnit.MILLISECONDS);
+ } catch (final IllegalArgumentException iae) {
+ throw new ProviderCreationException(String.format("The Expiration Duration '%s' is not a valid time duration", rawExpiration));
+ }
+
+ final LdapContextSource context = new LdapContextSource();
+
+ final Map<String, Object> baseEnvironment = new HashMap<>();
+
+ // connect/read time out
+ setTimeout(configurationContext, baseEnvironment, "Connect Timeout", "com.sun.jndi.ldap.connect.timeout");
+ setTimeout(configurationContext, baseEnvironment, "Read Timeout", "com.sun.jndi.ldap.read.timeout");
+
+ // set the base environment is necessary
+ if (!baseEnvironment.isEmpty()) {
+ context.setBaseEnvironmentProperties(baseEnvironment);
+ }
+
+ // authentication strategy
+ final String rawAuthenticationStrategy = configurationContext.getProperty("Authentication Strategy");
+ final LdapAuthenticationStrategy authenticationStrategy;
+ try {
+ authenticationStrategy = LdapAuthenticationStrategy.valueOf(rawAuthenticationStrategy);
+ } catch (final IllegalArgumentException iae) {
+ throw new ProviderCreationException(String.format("Unrecognized authentication strategy '%s'. Possible values are [%s]",
+ rawAuthenticationStrategy, StringUtils.join(LdapAuthenticationStrategy.values(), ", ")));
+ }
+
+ switch (authenticationStrategy) {
+ case ANONYMOUS:
+ context.setAnonymousReadOnly(true);
+ break;
+ default:
+ final String userDn = configurationContext.getProperty("Manager DN");
+ final String password = configurationContext.getProperty("Manager Password");
+
+ context.setUserDn(userDn);
+ context.setPassword(password);
+
+ switch (authenticationStrategy) {
+ case SIMPLE:
+ context.setAuthenticationStrategy(new SimpleDirContextAuthenticationStrategy());
+ break;
+ case START_TLS:
+ final AbstractTlsDirContextAuthenticationStrategy tlsAuthenticationStrategy = new DefaultTlsDirContextAuthenticationStrategy();
+
+ // shutdown gracefully
+ final String rawShutdownGracefully = configurationContext.getProperty("TLS - Shutdown Gracefully");
+ if (StringUtils.isNotBlank(rawShutdownGracefully)) {
+ final boolean shutdownGracefully = Boolean.TRUE.toString().equalsIgnoreCase(rawShutdownGracefully);
+ tlsAuthenticationStrategy.setShutdownTlsGracefully(shutdownGracefully);
+ }
+
+ final String rawKeystore = configurationContext.getProperty("TLS - Keystore");
+ final String rawKeystorePassword = configurationContext.getProperty("TLS - Keystore Password");
+ final String rawKeystoreType = configurationContext.getProperty("TLS - Keystore Type");
+ final String rawTruststore = configurationContext.getProperty("TLS - Truststore");
+ final String rawTruststorePassword = configurationContext.getProperty("TLS - Truststore Password");
+ final String rawTruststoreType = configurationContext.getProperty("TLS - Truststore Type");
+ final String rawClientAuth = configurationContext.getProperty("TLS - Client Auth");
+ final String rawProtocol = configurationContext.getProperty("TLS - Protocol");
+
+ final ClientAuth clientAuth;
+ if (StringUtils.isBlank(rawClientAuth)) {
+ clientAuth = ClientAuth.NONE;
+ } else {
+ try {
+ clientAuth = ClientAuth.valueOf(rawClientAuth);
+ } catch (final IllegalArgumentException iae) {
+ throw new ProviderCreationException(String.format("Unrecognized client auth '%s'. Possible values are [%s]",
+ rawClientAuth, StringUtils.join(ClientAuth.values(), ", ")));
+ }
+ }
+
+ try {
+ final SSLContext sslContext;
+ if (StringUtils.isBlank(rawKeystore)) {
+ sslContext = SslContextFactory.createTrustSslContext(rawTruststore, rawTruststorePassword.toCharArray(), rawTruststoreType, rawProtocol);
+ } else {
+ if (StringUtils.isBlank(rawTruststore)) {
+ sslContext = SslContextFactory.createSslContext(rawKeystore, rawKeystorePassword.toCharArray(), rawKeystoreType, rawProtocol);
+ } else {
+ sslContext = SslContextFactory.createSslContext(rawKeystore, rawKeystorePassword.toCharArray(), rawKeystoreType,
+ rawTruststore, rawTruststorePassword.toCharArray(), rawTruststoreType, clientAuth, rawProtocol);
+ }
+ }
+ tlsAuthenticationStrategy.setSslSocketFactory(sslContext.getSocketFactory());
+ } catch (final KeyStoreException | NoSuchAlgorithmException | CertificateException | UnrecoverableKeyException | KeyManagementException | IOException e) {
+ throw new ProviderCreationException(e.getMessage(), e);
+ }
+
+ context.setAuthenticationStrategy(tlsAuthenticationStrategy);
+ break;
+ }
+ break;
+ }
+
+ // referrals
+ final String rawReferralStrategy = configurationContext.getProperty("Referral Strategy");
+
+ final ReferralStrategy referralStrategy;
+ try {
+ referralStrategy = ReferralStrategy.valueOf(rawReferralStrategy);
+ } catch (final IllegalArgumentException iae) {
+ throw new ProviderCreationException(String.format("Unrecgonized authentication strategy '%s'. Possible values are [%s]",
+ rawAuthenticationStrategy, StringUtils.join(ReferralStrategy.values(), ", ")));
+ }
+
+ context.setReferral(referralStrategy.toString());
+
+ // url
+ final String url = configurationContext.getProperty("Url");
+
+ if (StringUtils.isBlank(url)) {
+ throw new ProviderCreationException("LDAP identity provider 'Url' must be specified.");
+ }
+
+ // connection
+ context.setUrl(url);
+
+ // search criteria
+ final String userSearchBase = configurationContext.getProperty("User Search Base");
+ final String userSearchFilter = configurationContext.getProperty("User Search Filter");
+
+ if (StringUtils.isBlank(userSearchBase) || StringUtils.isBlank(userSearchFilter)) {
+ throw new ProviderCreationException("LDAP identity provider 'User Search Base' and 'User Search Filter' must be specified.");
+ }
+
+ final LdapUserSearch userSearch = new FilterBasedLdapUserSearch(userSearchBase, userSearchFilter, context);
+
+ // bind
+ final BindAuthenticator authenticator = new BindAuthenticator(context);
+ authenticator.setUserSearch(userSearch);
+
+ try {
+ // handling initializing beans
+ context.afterPropertiesSet();
+ authenticator.afterPropertiesSet();
+ } catch (final Exception e) {
+ throw new ProviderCreationException(e.getMessage(), e);
+ }
+
+ // create the underlying provider
+ provider = new LdapAuthenticationProvider(authenticator);
+ }
+
+ private void setTimeout(final LoginIdentityProviderConfigurationContext configurationContext,
+ final Map<String, Object> baseEnvironment,
+ final String configurationProperty,
+ final String environmentKey) {
+
+ final String rawTimeout = configurationContext.getProperty(configurationProperty);
+ if (StringUtils.isNotBlank(rawTimeout)) {
+ try {
+ final Long timeout = FormatUtils.getTimeDuration(rawTimeout, TimeUnit.MILLISECONDS);
+ baseEnvironment.put(environmentKey, timeout.toString());
+ } catch (final IllegalArgumentException iae) {
+ throw new ProviderCreationException(String.format("The %s '%s' is not a valid time duration", configurationProperty, rawTimeout));
+ }
+ }
+ }
+
+ @Override
+ public final AuthenticationResponse authenticate(final LoginCredentials credentials) throws InvalidLoginCredentialsException, IdentityAccessException {
+ if (provider == null) {
+ throw new IdentityAccessException("The LDAP authentication provider is not initialized.");
+ }
+
+ try {
+ // perform the authentication
+ final UsernamePasswordAuthenticationToken token = new UsernamePasswordAuthenticationToken(credentials.getUsername(), credentials.getPassword());
+ final Authentication authentication = provider.authenticate(token);
+
+ // attempt to get the ldap user details to get the DN
+ if (authentication.getPrincipal() instanceof LdapUserDetails) {
+ final LdapUserDetails userDetails = (LdapUserDetails) authentication.getPrincipal();
+ return new AuthenticationResponse(userDetails.getDn(), credentials.getUsername(), expiration, issuer);
+ } else {
+ return new AuthenticationResponse(authentication.getName(), credentials.getUsername(), expiration, issuer);
+ }
+ } catch (final CommunicationException | AuthenticationServiceException e) {
+ logger.error(e.getMessage());
+ if (logger.isDebugEnabled()) {
+ logger.debug(StringUtils.EMPTY, e);
+ }
+ throw new IdentityAccessException("Unable to query the configured directory server. See the logs for additional details.", e);
+ } catch (final BadCredentialsException bce) {
+ throw new InvalidLoginCredentialsException(bce.getMessage(), bce);
+ }
+ }
+
+ @Override
+ public final void preDestruction() throws ProviderDestructionException {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-ldap-iaa-providers-bundle/nifi-ldap-iaa-providers/src/main/java/org/apache/nifi/ldap/ReferralStrategy.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-ldap-iaa-providers-bundle/nifi-ldap-iaa-providers/src/main/java/org/apache/nifi/ldap/ReferralStrategy.java b/nifi-nar-bundles/nifi-ldap-iaa-providers-bundle/nifi-ldap-iaa-providers/src/main/java/org/apache/nifi/ldap/ReferralStrategy.java
new file mode 100644
index 0000000..f4c5131
--- /dev/null
+++ b/nifi-nar-bundles/nifi-ldap-iaa-providers-bundle/nifi-ldap-iaa-providers/src/main/java/org/apache/nifi/ldap/ReferralStrategy.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.ldap;
+
+/**
+ *
+ */
+public enum ReferralStrategy {
+
+ FOLLOW("follow"),
+ INGORE("ignore"),
+ THROW("throw");
+
+ private final String value;
+
+ private ReferralStrategy(String value) {
+ this.value = value;
+ }
+
+ @Override
+ public String toString() {
+ return value;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-ldap-iaa-providers-bundle/nifi-ldap-iaa-providers/src/main/resources/META-INF/services/org.apache.nifi.authentication.LoginIdentityProvider
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-ldap-iaa-providers-bundle/nifi-ldap-iaa-providers/src/main/resources/META-INF/services/org.apache.nifi.authentication.LoginIdentityProvider b/nifi-nar-bundles/nifi-ldap-iaa-providers-bundle/nifi-ldap-iaa-providers/src/main/resources/META-INF/services/org.apache.nifi.authentication.LoginIdentityProvider
new file mode 100644
index 0000000..b5ca1fe
--- /dev/null
+++ b/nifi-nar-bundles/nifi-ldap-iaa-providers-bundle/nifi-ldap-iaa-providers/src/main/resources/META-INF/services/org.apache.nifi.authentication.LoginIdentityProvider
@@ -0,0 +1,15 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+org.apache.nifi.ldap.LdapProvider
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-ldap-iaa-providers-bundle/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-ldap-iaa-providers-bundle/pom.xml b/nifi-nar-bundles/nifi-ldap-iaa-providers-bundle/pom.xml
new file mode 100644
index 0000000..e038c2c
--- /dev/null
+++ b/nifi-nar-bundles/nifi-ldap-iaa-providers-bundle/pom.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-nar-bundles</artifactId>
+ <version>0.3.1-SNAPSHOT</version>
+ </parent>
+ <artifactId>nifi-ldap-iaa-providers-bundle</artifactId>
+ <packaging>pom</packaging>
+ <modules>
+ <module>nifi-ldap-iaa-providers</module>
+ <module>nifi-ldap-iaa-providers-nar</module>
+ </modules>
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-ldap-iaa-providers</artifactId>
+ <version>0.3.1-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/pom.xml b/nifi-nar-bundles/pom.xml
index 4c0925f..a9c7728 100644
--- a/nifi-nar-bundles/pom.xml
+++ b/nifi-nar-bundles/pom.xml
@@ -48,6 +48,7 @@
<module>nifi-avro-bundle</module>
<module>nifi-couchbase-bundle</module>
<module>nifi-azure-bundle</module>
+ <module>nifi-ldap-iaa-providers-bundle</module>
</modules>
<dependencyManagement>
<dependencies>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 0708812..249f01c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1,14 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
- license agreements. See the NOTICE file distributed with this work for additional
- information regarding copyright ownership. The ASF licenses this file to
- You under the Apache License, Version 2.0 (the "License"); you may not use
- this file except in compliance with the License. You may obtain a copy of
- the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
- by applicable law or agreed to in writing, software distributed under the
- License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
- OF ANY KIND, either express or implied. See the License for the specific
- language governing permissions and limitations under the License. -->
+license agreements. See the NOTICE file distributed with this work for additional
+information regarding copyright ownership. The ASF licenses this file to
+You under the Apache License, Version 2.0 (the "License"); you may not use
+this file except in compliance with the License. You may obtain a copy of
+the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+by applicable law or agreed to in writing, software distributed under the
+License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+OF ANY KIND, either express or implied. See the License for the specific
+language governing permissions and limitations under the License. -->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
@@ -91,7 +91,7 @@
<jetty.version>9.2.11.v20150529</jetty.version>
<lucene.version>4.10.4</lucene.version>
<spring.version>4.1.6.RELEASE</spring.version>
- <spring.security.version>3.2.7.RELEASE</spring.security.version>
+ <spring.security.version>4.0.3.RELEASE</spring.security.version>
<jersey.version>1.19</jersey.version>
<hadoop.version>2.6.2</hadoop.version>
<hadoop.guava.version>12.0.1</hadoop.guava.version>
@@ -290,8 +290,8 @@
<version>2.2.1</version>
<exclusions>
<!-- | Exclude the quartz 2.2.1 bundled version of c3p0 because it is
- lgpl licensed | We also don't use the JDBC related features of quartz for
- which the dependency would matter -->
+ lgpl licensed | We also don't use the JDBC related features of quartz for
+ which the dependency would matter -->
<exclusion>
<groupId>c3p0</groupId>
<artifactId>c3p0</artifactId>
@@ -361,7 +361,7 @@
<version>${spring.version}</version>
<exclusions>
<!-- <artifactId>jcl-over-slf4j</artifactId> is used in dependencies
- section -->
+ section -->
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
@@ -504,6 +504,29 @@
</exclusions>
</dependency>
<dependency>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-ldap</artifactId>
+ <version>${spring.security.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-beans</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-context</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-tx</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
<groupId>org.aspectj</groupId>
<artifactId>aspectjweaver</artifactId>
<version>1.8.5</version>
@@ -977,6 +1000,12 @@
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-ldap-iaa-providers-nar</artifactId>
+ <version>0.3.1-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
<artifactId>nifi-properties</artifactId>
<version>0.4.0-SNAPSHOT</version>
</dependency>
@@ -1262,7 +1291,7 @@
<module name="OuterTypeFilename" />
<module name="LineLength">
<!-- needs extra, because Eclipse formatter ignores the ending left
- brace -->
+ brace -->
<property name="max" value="200" />
<property name="ignorePattern" value="^package.*|^import.*|a href|href|http://|https://|ftp://" />
</module>
@@ -1352,11 +1381,11 @@
<profiles>
<profile>
<!-- Checks style and licensing requirements. This is a good idea to run
- for contributions and for the release process. While it would be nice to
- run always these plugins can considerably slow the build and have proven
- to create unstable builds in our multi-module project and when building using
- multiple threads. The stability issues seen with Checkstyle in multi-module
- builds include false-positives and false negatives. -->
+ for contributions and for the release process. While it would be nice to
+ run always these plugins can considerably slow the build and have proven
+ to create unstable builds in our multi-module project and when building using
+ multiple threads. The stability issues seen with Checkstyle in multi-module
+ builds include false-positives and false negatives. -->
<id>contrib-check</id>
<build>
<plugins>
[41/50] [abbrv] nifi git commit: NIFI-655: - Refactoring web security
to use Spring Security Java Configuration. - Introducing security in Web UI
in order to get JWT.
Posted by mc...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/login/nf-login.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/login/nf-login.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/login/nf-login.js
new file mode 100644
index 0000000..6cefd4f
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/login/nf-login.js
@@ -0,0 +1,302 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* global nf, top */
+
+$(document).ready(function () {
+ nf.Login.init();
+});
+
+nf.Login = (function () {
+
+ var supportsAnonymous = false;
+
+ var config = {
+ urls: {
+ identity: '../nifi-api/controller/identity',
+ users: '../nifi-api/controller/users',
+ token: '../nifi-api/access/token',
+ accessStatus: '../nifi-api/access',
+ accessConfig: '../nifi-api/access/config'
+ }
+ };
+
+ var initializeMessage = function () {
+ $('#login-message-container').show();
+ };
+
+ var showLogin = function () {
+ // reset the forms
+ $('#username').val('');
+ $('#password').val('');
+ $('#login-submission-button').text('Log in');
+
+ // update the form visibility
+ $('#login-container').show();
+ $('#nifi-registration-container').hide();
+
+ // set the focus
+ $('#username').focus();
+ };
+
+ var initializeNiFiRegistration = function () {
+ $('#nifi-registration-justification').count({
+ charCountField: '#remaining-characters'
+ });
+
+ // toggle between signup and login
+ $('#login-to-account-link').on('click', function () {
+ showLogin();
+ });
+ };
+
+ var showNiFiRegistration = function () {
+ // reset the forms
+ $('#login-submission-button').text('Submit');
+ $('#nifi-registration-justification').val('');
+
+ // update the form visibility
+ $('#login-container').hide();
+ $('#nifi-registration-container').show();
+ };
+
+ var initializeSubmission = function () {
+ $('#login-submission-button').on('click', function () {
+ if ($('#login-container').is(':visible')) {
+ login();
+ } else if ($('#nifi-registration-container').is(':visible')) {
+ submitJustification();
+ }
+ });
+
+ $('#login-submission-container').show();
+ };
+
+ var login = function () {
+ // show the logging message...
+ $('#login-progress-label').text('Logging in...');
+ $('#login-progress-container').show();
+ $('#login-submission-container').hide();
+
+ // login submit
+ $.ajax({
+ type: 'POST',
+ url: config.urls.token,
+ data: {
+ 'username': $('#username').val(),
+ 'password': $('#password').val()
+ }
+ }).done(function (jwt) {
+ // get the payload and store the token with the appropirate expiration
+ var token = nf.Common.getJwtPayload(jwt);
+ var expiration = parseInt(token['exp'], 10) * nf.Common.MILLIS_PER_SECOND;
+ nf.Storage.setItem('jwt', jwt, expiration);
+
+ // check to see if they actually have access now
+ $.ajax({
+ type: 'GET',
+ url: config.urls.identity,
+ dataType: 'json'
+ }).done(function (response) {
+ if (response.identity === 'anonymous') {
+ showLogoutLink();
+
+ // schedule automatic token refresh
+ nf.Common.scheduleTokenRefresh();
+
+ // show the user
+ $('#nifi-user-submit-justification').text(token['preferred_username']);
+
+ // show the registration form
+ initializeNiFiRegistration();
+ showNiFiRegistration();
+
+ // update the form visibility
+ $('#login-submission-container').show();
+ $('#login-progress-container').hide();
+ } else {
+ // reload as appropriate - no need to schedule token refresh as the page is reloading
+ if (top !== window) {
+ parent.window.location = '/nifi';
+ } else {
+ window.location = '/nifi';
+ }
+ }
+ }).fail(function (xhr, status, error) {
+ showLogoutLink();
+
+ // schedule automatic token refresh
+ nf.Common.scheduleTokenRefresh();
+
+ // show the user
+ $('#nifi-user-submit-justification').text(token['preferred_username']);
+
+ if (xhr.status === 401) {
+ initializeNiFiRegistration();
+ showNiFiRegistration();
+
+ // update the form visibility
+ $('#login-submission-container').show();
+ $('#login-progress-container').hide();
+ } else {
+ $('#login-message-title').text('Unable to log in');
+ $('#login-message').text(xhr.responseText);
+
+ // update visibility
+ $('#login-container').hide();
+ $('#login-submission-container').hide();
+ $('#login-progress-container').hide();
+ $('#login-message-container').show();
+ }
+ });
+ }).fail(function (xhr, status, error) {
+ nf.Dialog.showOkDialog({
+ dialogContent: nf.Common.escapeHtml(xhr.responseText),
+ overlayBackground: false
+ });
+
+ // update the form visibility
+ $('#login-submission-container').show();
+ $('#login-progress-container').hide();
+ });
+ };
+
+ var submitJustification = function () {
+ // show the logging message...
+ $('#login-progress-label').text('Submitting...');
+ $('#login-progress-container').show();
+ $('#login-submission-container').hide();
+
+ // attempt to create the nifi account registration
+ $.ajax({
+ type: 'POST',
+ url: config.urls.users,
+ data: {
+ 'justification': $('#nifi-registration-justification').val()
+ }
+ }).done(function (response) {
+ var markup = 'An administrator will process your request shortly.';
+ if (supportsAnonymous === true) {
+ markup += '<br/><br/>In the meantime you can continue accessing anonymously.';
+ }
+
+ $('#login-message-title').text('Thanks!');
+ $('#login-message').html(markup);
+ }).fail(function (xhr, status, error) {
+ $('#login-message-title').text('Unable to submit justification');
+ $('#login-message').text(xhr.responseText);
+ }).always(function () {
+ // update form visibility
+ $('#nifi-registration-container').hide();
+ $('#login-submission-container').hide();
+ $('#login-progress-container').hide();
+ $('#login-message-container').show();
+ });
+ };
+
+ var showLogoutLink = function () {
+ nf.Common.showLogoutLink();
+ };
+
+ return {
+ /**
+ * Initializes the login page.
+ */
+ init: function () {
+ nf.Storage.init();
+
+ if (nf.Storage.getItem('jwt') !== null) {
+ showLogoutLink();
+ }
+
+ // access status
+ var accessStatus = $.ajax({
+ type: 'GET',
+ url: config.urls.accessStatus,
+ dataType: 'json'
+ }).fail(function (xhr, status, error) {
+ $('#login-message-title').text('Unable to check Access Status');
+ $('#login-message').text(xhr.responseText);
+ initializeMessage();
+ });
+
+ // access config
+ var accessConfigXhr = $.ajax({
+ type: 'GET',
+ url: config.urls.accessConfig,
+ dataType: 'json'
+ });
+
+ $.when(accessStatus, accessConfigXhr).done(function (accessStatusResult, accessConfigResult) {
+ var accessStatusResponse = accessStatusResult[0];
+ var accessStatus = accessStatusResponse.accessStatus;
+
+ var accessConfigResponse = accessConfigResult[0];
+ var accessConfig = accessConfigResponse.config;
+
+ // record whether this NiFi supports anonymous access
+ supportsAnonymous = accessConfig.supportsAnonymous;
+
+ // possible login states
+ var needsLogin = false;
+ var needsNiFiRegistration = false;
+ var showMessage = false;
+
+ // handle the status appropriately
+ if (accessStatus.status === 'UNKNOWN') {
+ needsLogin = true;
+ } else if (accessStatus.status === 'UNREGISTERED') {
+ needsNiFiRegistration = true;
+
+ $('#nifi-user-submit-justification').text(accessStatus.username);
+ } else if (accessStatus.status === 'NOT_ACTIVE') {
+ showMessage = true;
+
+ $('#login-message-title').text('Access Denied');
+ $('#login-message').text(accessStatus.message);
+ } else if (accessStatus.status === 'ACTIVE') {
+ showMessage = true;
+
+ $('#login-message-title').text('Success');
+ $('#login-message').text('Your account is active and you are already logged in.');
+ }
+
+ // if login is required, verify its supported
+ if (accessConfig.supportsLogin === false && needsLogin === true) {
+ $('#login-message-title').text('Access Denied');
+ $('#login-message').text('This NiFi is not configured to support login.');
+ showMessage = true;
+ needsLogin = false;
+ }
+
+ // initialize the page as appropriate
+ if (showMessage === true) {
+ initializeMessage();
+ } else if (needsLogin === true) {
+ showLogin();
+ } else if (needsNiFiRegistration === true) {
+ initializeNiFiRegistration();
+ showNiFiRegistration();
+ }
+
+ if (needsLogin === true || needsNiFiRegistration === true) {
+ initializeSubmission();
+ }
+ });
+ }
+ };
+}());
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/nf-common.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/nf-common.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/nf-common.js
index 06c34f9..321044f 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/nf-common.js
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/nf-common.js
@@ -50,948 +50,1110 @@ $(document).ready(function () {
// hide the loading indicator
$('div.loading-container').removeClass('ajax-loading');
});
-
- // initialize the tooltips
- $('img.setting-icon').qtip(nf.Common.config.tooltipConfig);
-});
-
-// Define a common utility class used across the entire application.
-nf.Common = {
- config: {
- sensitiveText: 'Sensitive value set',
- tooltipConfig: {
- style: {
- classes: 'nifi-tooltip'
- },
- show: {
- solo: true,
- effect: false
- },
- hide: {
- effect: false
- },
- position: {
- at: 'top right',
- my: 'bottom left'
- }
- }
- },
-
- /**
- * Determines if the current broswer supports SVG.
- */
- SUPPORTS_SVG: !!document.createElementNS && !!document.createElementNS('http://www.w3.org/2000/svg', 'svg').createSVGRect,
-
- /**
- * The authorities for the current user.
- */
- authorities: undefined,
- /**
- * Sets the authorities for the current user.
- *
- * @argument {array} roles The current users authorities
- */
- setAuthorities: function (roles) {
- nf.Common.authorities = roles;
- },
-
- /**
- * Loads a script at the specified URL. Supports caching the script on the browser.
- *
- * @param {string} url
- */
- cachedScript: function (url) {
- return $.ajax({
- dataType: 'script',
- cache: true,
- url: url
- });
- },
-
- /**
- * Determines whether the current user can access provenance.
- *
- * @returns {boolean}
- */
- canAccessProvenance: function () {
- var canAccessProvenance = false;
- if (nf.Common.isDefinedAndNotNull(nf.Common.authorities)) {
- $.each(nf.Common.authorities, function (i, authority) {
- if (authority === 'ROLE_PROVENANCE') {
- canAccessProvenance = true;
- return false;
- }
- });
- }
- return canAccessProvenance;
- },
-
- /**
- * Returns whether or not the current user is a DFM.
- */
- isDFM: function () {
- var dfm = false;
- if (nf.Common.isDefinedAndNotNull(nf.Common.authorities)) {
- $.each(nf.Common.authorities, function (i, authority) {
- if (authority === 'ROLE_DFM') {
- dfm = true;
+ // include jwt when possible
+ $.ajaxSetup({
+ 'beforeSend': function(xhr) {
+ var hadToken = nf.Storage.hasItem('jwt');
+
+ // get the token to include in all requests
+ var token = nf.Storage.getItem('jwt');
+ if (token !== null) {
+ xhr.setRequestHeader('Authorization', 'Bearer ' + token);
+ } else {
+ // if the current user was logged in with a token and the token just expired, reload
+ if (hadToken === true) {
return false;
}
- });
+ }
}
- return dfm;
- },
+ });
+
+ // initialize the tooltips
+ $('img.setting-icon').qtip(nf.Common.config.tooltipConfig);
- /**
- * Returns whether or not the current user is a DFM.
- */
- isAdmin: function () {
- var admin = false;
- if (nf.Common.isDefinedAndNotNull(nf.Common.authorities)) {
- $.each(nf.Common.authorities, function (i, authority) {
- if (authority === 'ROLE_ADMIN') {
- admin = true;
- return false;
- }
- });
- }
- return admin;
- },
+ // shows the logout link in the message-pane when appropriate and schedule token refresh
+ if (nf.Storage.getItem('jwt') !== null) {
+ $('#user-logout-container').show();
+ nf.Common.scheduleTokenRefresh();
+ }
- /**
- * Adds a mouse over effect for the specified selector using
- * the specified styles.
- *
- * @argument {string} selector The selector for the element to add a hover effect for
- * @argument {string} normalStyle The css style for the normal state
- * @argument {string} overStyle The css style for the over state
- */
- addHoverEffect: function (selector, normalStyle, overStyle) {
- $(document).on('mouseenter', selector, function () {
- $(this).removeClass(normalStyle).addClass(overStyle);
- }).on('mouseleave', selector, function () {
- $(this).removeClass(overStyle).addClass(normalStyle);
- });
- return $(selector).addClass(normalStyle);
- },
+ // handle logout
+ $('#user-logout').on('click', function () {
+ nf.Storage.removeItem('jwt');
+ window.location = '/nifi/login';
+ });
+});
+
+// Define a common utility class used across the entire application.
+nf.Common = (function () {
+ // interval for cancelling token refresh when necessary
+ var tokenRefreshInterval = null;
- /**
- * Method for handling ajax errors.
- *
- * @argument {object} xhr The XmlHttpRequest
- * @argument {string} status The status of the request
- * @argument {string} error The error
- */
- handleAjaxError: function (xhr, status, error) {
- // show the account registration page if necessary
- if (xhr.status === 401 && $('#registration-pane').length) {
- // show the registration pane
- $('#registration-pane').show();
-
- // close the canvas
- nf.Common.closeCanvas();
- return;
- }
+ return {
+ ANONYMOUS_USER_TEXT: 'Anonymous user',
- // if an error occurs while the splash screen is visible close the canvas show the error message
- if ($('#splash').is(':visible')) {
- $('#message-title').text('An unexpected error has occurred');
- if ($.trim(xhr.responseText) === '') {
- $('#message-content').text('Please check the logs.');
- } else {
- $('#message-content').text(xhr.responseText);
+ config: {
+ sensitiveText: 'Sensitive value set',
+ tooltipConfig: {
+ style: {
+ classes: 'nifi-tooltip'
+ },
+ show: {
+ solo: true,
+ effect: false
+ },
+ hide: {
+ effect: false
+ },
+ position: {
+ at: 'top right',
+ my: 'bottom left'
+ }
}
+ },
- // show the error pane
- $('#message-pane').show();
+ /**
+ * Determines if the current broswer supports SVG.
+ */
+ SUPPORTS_SVG: !!document.createElementNS && !!document.createElementNS('http://www.w3.org/2000/svg', 'svg').createSVGRect,
- // close the canvas
- nf.Common.closeCanvas();
- return;
- }
-
- // status code 400, 404, and 409 are expected response codes for common errors.
- if (xhr.status === 400 || xhr.status === 404 || xhr.status === 409) {
- nf.Dialog.showOkDialog({
- dialogContent: nf.Common.escapeHtml(xhr.responseText),
- overlayBackground: false
+ /**
+ * The authorities for the current user.
+ */
+ authorities: undefined,
+
+ /**
+ * Sets the authorities for the current user.
+ *
+ * @argument {array} roles The current users authorities
+ */
+ setAuthorities: function (roles) {
+ nf.Common.authorities = roles;
+ },
+
+ /**
+ * Loads a script at the specified URL. Supports caching the script on the browser.
+ *
+ * @param {string} url
+ */
+ cachedScript: function (url) {
+ return $.ajax({
+ dataType: 'script',
+ cache: true,
+ url: url
});
- } else {
- if (xhr.status < 99 || xhr.status === 12007 || xhr.status === 12029) {
- var content = 'Please ensure the application is running and check the logs for any errors.';
- if (nf.Common.isDefinedAndNotNull(status)) {
- if (status === 'timeout') {
- content = 'Request has timed out. Please ensure the application is running and check the logs for any errors.';
- } else if (status === 'abort') {
- content = 'Request has been aborted.';
- } else if (status === 'No Transport') {
- content = 'Request transport mechanism failed. Please ensure the host where the application is running is accessible.';
+ },
+
+ /**
+ * Automatically refresh tokens by checking once an hour if its going to expire soon.
+ */
+ scheduleTokenRefresh: function () {
+ // if we are currently polling for token refresh, cancel it
+ if (tokenRefreshInterval !== null) {
+ clearInterval(tokenRefreshInterval);
+ }
+
+ // set the interval to one hour
+ var interval = nf.Common.MILLIS_PER_MINUTE;
+
+ var checkExpiration = function () {
+ var expiration = nf.Storage.getItemExpiration('jwt');
+
+ // ensure there is an expiration and token present
+ if (expiration !== null) {
+ var expirationDate = new Date(expiration);
+ var now = new Date();
+
+ // get the time remainging plus a little bonus time to reload the token
+ var timeRemaining = expirationDate.valueOf() - now.valueOf() - (30 * nf.Common.MILLIS_PER_SECOND);
+ if (timeRemaining < interval) {
+ if ($('#current-user').text() !== nf.Common.ANONYMOUS_USER_TEXT && !$('#anonymous-user-alert').is(':visible')) {
+ // if the token will expire before the next interval minus some bonus time, notify the user to re-login
+ $('#anonymous-user-alert').show().qtip($.extend({}, nf.Common.config.tooltipConfig, {
+ content: 'Your session will expire soon. Please log in again to avoid being automatically logged out.',
+ position: {
+ my: 'top right',
+ at: 'bottom left'
+ }
+ }));
+ }
}
}
- $('#message-title').text('Unable to communicate with NiFi');
- $('#message-content').text(content);
- } else if (xhr.status === 401) {
- $('#message-title').text('Unauthorized');
- if ($.trim(xhr.responseText) === '') {
- $('#message-content').text('Authorization is required to use this NiFi.');
+ };
+
+ // perform initial check
+ checkExpiration();
+
+ // schedule subsequent checks
+ tokenRefreshInterval = setInterval(checkExpiration, interval);
+ },
+
+ /**
+ * Sets the anonymous user label.
+ */
+ setAnonymousUserLabel: function () {
+ var anonymousUserAlert = $('#anonymous-user-alert');
+ if (anonymousUserAlert.data('qtip')) {
+ anonymousUserAlert.qtip('api').destroy(true);
+ }
+
+ // alert user's of anonymous access
+ anonymousUserAlert.show().qtip($.extend({}, nf.Common.config.tooltipConfig, {
+ content: 'You are accessing with limited authority. Log in or request an account to access with additional authority granted to you by an administrator.',
+ position: {
+ my: 'top right',
+ at: 'bottom left'
+ }
+ }));
+
+ // render the anonymous user text
+ $('#current-user').text(nf.Common.ANONYMOUS_USER_TEXT).show();
+ },
+
+ /**
+ * Extracts the subject from the specified jwt. If the jwt is not as expected
+ * an empty string is returned.
+ *
+ * @param {string} jwt
+ * @returns {string}
+ */
+ getJwtPayload: function (jwt) {
+ if (nf.Common.isDefinedAndNotNull(jwt)) {
+ var segments = jwt.split(/\./);
+ if (segments.length !== 3) {
+ return '';
+ }
+
+ var rawPayload = $.base64.atob(segments[1]);
+ var payload = JSON.parse(rawPayload);
+
+ if (nf.Common.isDefinedAndNotNull(payload)) {
+ return payload;
} else {
- $('#message-content').text(xhr.responseText);
+ return null;
}
- } else if (xhr.status === 403) {
- $('#message-title').text('Forbidden');
- if ($.trim(xhr.responseText) === '') {
- $('#message-content').text('Unable to authorize you to use this NiFi.');
+ }
+
+ return null;
+ },
+
+ /**
+ * Determines whether the current user can access provenance.
+ *
+ * @returns {boolean}
+ */
+ canAccessProvenance: function () {
+ var canAccessProvenance = false;
+ if (nf.Common.isDefinedAndNotNull(nf.Common.authorities)) {
+ $.each(nf.Common.authorities, function (i, authority) {
+ if (authority === 'ROLE_PROVENANCE') {
+ canAccessProvenance = true;
+ return false;
+ }
+ });
+ }
+ return canAccessProvenance;
+ },
+
+ /**
+ * Returns whether or not the current user is a DFM.
+ */
+ isDFM: function () {
+ var dfm = false;
+ if (nf.Common.isDefinedAndNotNull(nf.Common.authorities)) {
+ $.each(nf.Common.authorities, function (i, authority) {
+ if (authority === 'ROLE_DFM') {
+ dfm = true;
+ return false;
+ }
+ });
+ }
+ return dfm;
+ },
+
+ /**
+ * Returns whether or not the current user is a DFM.
+ */
+ isAdmin: function () {
+ var admin = false;
+ if (nf.Common.isDefinedAndNotNull(nf.Common.authorities)) {
+ $.each(nf.Common.authorities, function (i, authority) {
+ if (authority === 'ROLE_ADMIN') {
+ admin = true;
+ return false;
+ }
+ });
+ }
+ return admin;
+ },
+
+ /**
+ * Adds a mouse over effect for the specified selector using
+ * the specified styles.
+ *
+ * @argument {string} selector The selector for the element to add a hover effect for
+ * @argument {string} normalStyle The css style for the normal state
+ * @argument {string} overStyle The css style for the over state
+ */
+ addHoverEffect: function (selector, normalStyle, overStyle) {
+ $(document).on('mouseenter', selector, function () {
+ $(this).removeClass(normalStyle).addClass(overStyle);
+ }).on('mouseleave', selector, function () {
+ $(this).removeClass(overStyle).addClass(normalStyle);
+ });
+ return $(selector).addClass(normalStyle);
+ },
+
+ /**
+ * Method for handling ajax errors.
+ *
+ * @argument {object} xhr The XmlHttpRequest
+ * @argument {string} status The status of the request
+ * @argument {string} error The error
+ */
+ handleAjaxError: function (xhr, status, error) {
+ if (status === 'canceled') {
+ if ($('#splash').is(':visible')) {
+ $('#message-title').text('Session Expired');
+ $('#message-content').text('Your session has expired. Please reload to log in again.');
+
+ // show the error pane
+ $('#message-pane').show();
} else {
- $('#message-content').text(xhr.responseText);
+ nf.Dialog.showOkDialog({
+ dialogContent: 'Your session has expired. Please press Ok to log in again.',
+ overlayBackground: false,
+ okHandler: function () {
+ window.location = '/nifi';
+ }
+ });
}
- } else if (xhr.status === 500) {
- $('#message-title').text('An unexpected error has occurred');
- if ($.trim(xhr.responseText) === '') {
- $('#message-content').text('An error occurred communicating with the application core. Please check the logs and fix any configuration issues before restarting.');
+
+ // close the canvas
+ nf.Common.closeCanvas();
+ return;
+ }
+
+ // if an error occurs while the splash screen is visible close the canvas show the error message
+ if ($('#splash').is(':visible')) {
+ if (xhr.status === 401) {
+ $('#message-title').text('Unauthorized');
+ } else if (xhr.status === 403) {
+ $('#message-title').text('Access Denied');
} else {
- $('#message-content').text(xhr.responseText);
+ $('#message-title').text('An unexpected error has occurred');
}
- } else if (xhr.status === 200 || xhr.status === 201) {
- $('#message-title').text('Parse Error');
+
if ($.trim(xhr.responseText) === '') {
- $('#message-content').text('Unable to interpret response from NiFi.');
+ $('#message-content').text('Please check the logs.');
} else {
$('#message-content').text(xhr.responseText);
}
+
+ // show the error pane
+ $('#message-pane').show();
+
+ // close the canvas
+ nf.Common.closeCanvas();
+ return;
+ }
+
+ // status code 400, 404, and 409 are expected response codes for common errors.
+ if (xhr.status === 400 || xhr.status === 404 || xhr.status === 409) {
+ nf.Dialog.showOkDialog({
+ dialogContent: nf.Common.escapeHtml(xhr.responseText),
+ overlayBackground: false
+ });
} else {
- $('#message-title').text(xhr.status + ': Unexpected Response');
- $('#message-content').text('An unexpected error has occurred. Please check the logs.');
+ if (xhr.status < 99 || xhr.status === 12007 || xhr.status === 12029) {
+ var content = 'Please ensure the application is running and check the logs for any errors.';
+ if (nf.Common.isDefinedAndNotNull(status)) {
+ if (status === 'timeout') {
+ content = 'Request has timed out. Please ensure the application is running and check the logs for any errors.';
+ } else if (status === 'abort') {
+ content = 'Request has been aborted.';
+ } else if (status === 'No Transport') {
+ content = 'Request transport mechanism failed. Please ensure the host where the application is running is accessible.';
+ }
+ }
+ $('#message-title').text('Unable to communicate with NiFi');
+ $('#message-content').text(content);
+ } else if (xhr.status === 401) {
+ $('#message-title').text('Unauthorized');
+ if ($.trim(xhr.responseText) === '') {
+ $('#message-content').text('Authorization is required to use this NiFi.');
+ } else {
+ $('#message-content').text(xhr.responseText);
+ }
+ } else if (xhr.status === 403) {
+ $('#message-title').text('Access Denied');
+ if ($.trim(xhr.responseText) === '') {
+ $('#message-content').text('Unable to authorize you to use this NiFi.');
+ } else {
+ $('#message-content').text(xhr.responseText);
+ }
+ } else if (xhr.status === 500) {
+ $('#message-title').text('An unexpected error has occurred');
+ if ($.trim(xhr.responseText) === '') {
+ $('#message-content').text('An error occurred communicating with the application core. Please check the logs and fix any configuration issues before restarting.');
+ } else {
+ $('#message-content').text(xhr.responseText);
+ }
+ } else if (xhr.status === 200 || xhr.status === 201) {
+ $('#message-title').text('Parse Error');
+ if ($.trim(xhr.responseText) === '') {
+ $('#message-content').text('Unable to interpret response from NiFi.');
+ } else {
+ $('#message-content').text(xhr.responseText);
+ }
+ } else {
+ $('#message-title').text(xhr.status + ': Unexpected Response');
+ $('#message-content').text('An unexpected error has occurred. Please check the logs.');
+ }
+
+ // show the error pane
+ $('#message-pane').show();
+
+ // close the canvas
+ nf.Common.closeCanvas();
}
+ },
- // show the error pane
- $('#message-pane').show();
+ /**
+ * Closes the canvas by removing the splash screen and stats poller.
+ */
+ closeCanvas: function () {
+ nf.Common.showLogoutLink();
+
+ // ensure this javascript has been loaded in the nf canvas page
+ if (nf.Common.isDefinedAndNotNull(nf.Canvas)) {
+ // hide the splash screen if required
+ if ($('#splash').is(':visible')) {
+ nf.Canvas.hideSplash();
+ }
- // close the canvas
- nf.Common.closeCanvas();
- }
- },
-
- /**
- * Closes the canvas by removing the splash screen and stats poller.
- */
- closeCanvas: function () {
- // ensure this javascript has been loaded in the nf canvas page
- if (nf.Common.isDefinedAndNotNull(nf.Canvas)) {
- // hide the splash screen if required
- if ($('#splash').is(':visible')) {
- nf.Canvas.hideSplash();
+ // hide the context menu
+ nf.ContextMenu.hide();
+
+ // shut off the auto refresh
+ nf.Canvas.stopRevisionPolling();
+ nf.Canvas.stopStatusPolling();
}
+ },
- // hide the context menu
- nf.ContextMenu.hide();
+ /**
+ * Shows the logout link if appropriate.
+ */
+ showLogoutLink: function () {
+ if (nf.Storage.getItem('jwt') === null) {
+ $('#user-logout-container').hide();
+ } else {
+ $('#user-logout-container').show();
+ }
+ },
- // shut off the auto refresh
- nf.Canvas.stopRevisionPolling();
- nf.Canvas.stopStatusPolling();
- }
- },
-
- /**
- * Populates the specified field with the specified value. If the value is
- * undefined, the field will read 'No value set.' If the value is an empty
- * string, the field will read 'Empty string set.'
- *
- * @argument {string} target The dom Id of the target
- * @argument {string} value The value
- */
- populateField: function (target, value) {
- if (nf.Common.isUndefined(value) || nf.Common.isNull(value)) {
- return $('#' + target).addClass('unset').text('No value set');
- } else if (value === '') {
- return $('#' + target).addClass('blank').text('Empty string set');
- } else {
- return $('#' + target).text(value);
- }
- },
-
- /**
- * Clears the specified field. Removes any style that may have been applied
- * by a preceeding call to populateField.
- *
- * @argument {string} target The dom Id of the target
- */
- clearField: function (target) {
- return $('#' + target).removeClass('unset blank').text('');
- },
-
- /**
- * Cleans up any tooltips that have been created for the specified container.
- *
- * @param {jQuery} container
- * @param {string} tooltipTarget
- */
- cleanUpTooltips: function(container, tooltipTarget) {
- container.find(tooltipTarget).each(function () {
- var tip = $(this);
- if (tip.data('qtip')) {
- var api = tip.qtip('api');
- api.destroy(true);
- }
- });
- },
-
- /**
- * Removes all read only property detail dialogs.
- */
- removeAllPropertyDetailDialogs: function () {
- var propertyDetails = $('body').children('div.property-detail');
- propertyDetails.find('div.nfel-editor').nfeditor('destroy');
- propertyDetails.hide().remove();
- },
-
- /**
- * Formats the tooltip for the specified property.
- *
- * @param {object} propertyDescriptor The property descriptor
- * @param {object} propertyHistory The property history
- * @returns {string}
- */
- formatPropertyTooltip: function (propertyDescriptor, propertyHistory) {
- var tipContent = [];
-
- // show the property description if applicable
- if (nf.Common.isDefinedAndNotNull(propertyDescriptor)) {
- if (!nf.Common.isBlank(propertyDescriptor.description)) {
- tipContent.push(nf.Common.escapeHtml(propertyDescriptor.description));
- }
- if (!nf.Common.isBlank(propertyDescriptor.defaultValue)) {
- tipContent.push('<b>Default value:</b> ' + nf.Common.escapeHtml(propertyDescriptor.defaultValue));
- }
- if (!nf.Common.isBlank(propertyDescriptor.supportsEl)) {
- tipContent.push('<b>Supports expression language:</b> ' + nf.Common.escapeHtml(propertyDescriptor.supportsEl));
+ /**
+ * Populates the specified field with the specified value. If the value is
+ * undefined, the field will read 'No value set.' If the value is an empty
+ * string, the field will read 'Empty string set.'
+ *
+ * @argument {string} target The dom Id of the target
+ * @argument {string} value The value
+ */
+ populateField: function (target, value) {
+ if (nf.Common.isUndefined(value) || nf.Common.isNull(value)) {
+ return $('#' + target).addClass('unset').text('No value set');
+ } else if (value === '') {
+ return $('#' + target).addClass('blank').text('Empty string set');
+ } else {
+ return $('#' + target).text(value);
}
- }
+ },
- if (nf.Common.isDefinedAndNotNull(propertyHistory)) {
- if (!nf.Common.isEmpty(propertyHistory.previousValues)) {
- var history = [];
- $.each(propertyHistory.previousValues, function (_, previousValue) {
- history.push('<li>' + nf.Common.escapeHtml(previousValue.previousValue) + ' - ' + nf.Common.escapeHtml(previousValue.timestamp) + ' (' + nf.Common.escapeHtml(previousValue.userName) + ')</li>');
- });
- tipContent.push('<b>History:</b><ul class="property-info">' + history.join('') + '</ul>');
+ /**
+ * Clears the specified field. Removes any style that may have been applied
+ * by a preceeding call to populateField.
+ *
+ * @argument {string} target The dom Id of the target
+ */
+ clearField: function (target) {
+ return $('#' + target).removeClass('unset blank').text('');
+ },
+
+ /**
+ * Cleans up any tooltips that have been created for the specified container.
+ *
+ * @param {jQuery} container
+ * @param {string} tooltipTarget
+ */
+ cleanUpTooltips: function(container, tooltipTarget) {
+ container.find(tooltipTarget).each(function () {
+ var tip = $(this);
+ if (tip.data('qtip')) {
+ var api = tip.qtip('api');
+ api.destroy(true);
+ }
+ });
+ },
+
+ /**
+ * Removes all read only property detail dialogs.
+ */
+ removeAllPropertyDetailDialogs: function () {
+ var propertyDetails = $('body').children('div.property-detail');
+ propertyDetails.find('div.nfel-editor').nfeditor('destroy');
+ propertyDetails.hide().remove();
+ },
+
+ /**
+ * Formats the tooltip for the specified property.
+ *
+ * @param {object} propertyDescriptor The property descriptor
+ * @param {object} propertyHistory The property history
+ * @returns {string}
+ */
+ formatPropertyTooltip: function (propertyDescriptor, propertyHistory) {
+ var tipContent = [];
+
+ // show the property description if applicable
+ if (nf.Common.isDefinedAndNotNull(propertyDescriptor)) {
+ if (!nf.Common.isBlank(propertyDescriptor.description)) {
+ tipContent.push(nf.Common.escapeHtml(propertyDescriptor.description));
+ }
+ if (!nf.Common.isBlank(propertyDescriptor.defaultValue)) {
+ tipContent.push('<b>Default value:</b> ' + nf.Common.escapeHtml(propertyDescriptor.defaultValue));
+ }
+ if (!nf.Common.isBlank(propertyDescriptor.supportsEl)) {
+ tipContent.push('<b>Supports expression language:</b> ' + nf.Common.escapeHtml(propertyDescriptor.supportsEl));
+ }
}
- }
- if (tipContent.length > 0) {
- return tipContent.join('<br/><br/>');
- } else {
- return null;
- }
- },
-
- /**
- * Formats the specified property (name and value) accordingly.
- *
- * @argument {string} name The name of the property
- * @argument {string} value The value of the property
- */
- formatProperty: function (name, value) {
- return '<div><span class="label">' + nf.Common.formatValue(name) + ': </span>' + nf.Common.formatValue(value) + '</div>';
- },
-
- /**
- * Formats the specified value accordingly.
- *
- * @argument {string} value The value of the property
- */
- formatValue: function (value) {
- if (nf.Common.isDefinedAndNotNull(value)) {
- if (value === '') {
- return '<span class="blank">Empty string set</span>';
+ if (nf.Common.isDefinedAndNotNull(propertyHistory)) {
+ if (!nf.Common.isEmpty(propertyHistory.previousValues)) {
+ var history = [];
+ $.each(propertyHistory.previousValues, function (_, previousValue) {
+ history.push('<li>' + nf.Common.escapeHtml(previousValue.previousValue) + ' - ' + nf.Common.escapeHtml(previousValue.timestamp) + ' (' + nf.Common.escapeHtml(previousValue.userName) + ')</li>');
+ });
+ tipContent.push('<b>History:</b><ul class="property-info">' + history.join('') + '</ul>');
+ }
+ }
+
+ if (tipContent.length > 0) {
+ return tipContent.join('<br/><br/>');
} else {
- return nf.Common.escapeHtml(value);
+ return null;
}
- } else {
- return '<span class="unset">No value set</span>';
- }
- },
-
- /**
- * HTML escapes the specified string. If the string is null
- * or undefined, an empty string is returned.
- *
- * @returns {string}
- */
- escapeHtml: (function () {
- var entityMap = {
- '&': '&',
- '<': '<',
- '>': '>',
- '"': '"',
- "'": ''',
- '/': '/'
- };
-
- return function (string) {
- if (nf.Common.isDefinedAndNotNull(string)) {
- return String(string).replace(/[&<>"'\/]/g, function (s) {
- return entityMap[s];
- });
+ },
+
+ /**
+ * Formats the specified property (name and value) accordingly.
+ *
+ * @argument {string} name The name of the property
+ * @argument {string} value The value of the property
+ */
+ formatProperty: function (name, value) {
+ return '<div><span class="label">' + nf.Common.formatValue(name) + ': </span>' + nf.Common.formatValue(value) + '</div>';
+ },
+
+ /**
+ * Formats the specified value accordingly.
+ *
+ * @argument {string} value The value of the property
+ */
+ formatValue: function (value) {
+ if (nf.Common.isDefinedAndNotNull(value)) {
+ if (value === '') {
+ return '<span class="blank">Empty string set</span>';
+ } else {
+ return nf.Common.escapeHtml(value);
+ }
} else {
- return '';
+ return '<span class="unset">No value set</span>';
}
- };
- }()),
-
- /**
- * Determines if the specified property is sensitive.
- *
- * @argument {object} propertyDescriptor The property descriptor
- */
- isSensitiveProperty: function (propertyDescriptor) {
- if (nf.Common.isDefinedAndNotNull(propertyDescriptor)) {
- return propertyDescriptor.sensitive === true;
- } else {
- return false;
- }
- },
-
- /**
- * Determines if the specified property is required.
- *
- * @param {object} propertyDescriptor The property descriptor
- */
- isRequiredProperty: function (propertyDescriptor) {
- if (nf.Common.isDefinedAndNotNull(propertyDescriptor)) {
- return propertyDescriptor.required === true;
- } else {
- return false;
- }
- },
-
- /**
- * Determines if the specified property is required.
- *
- * @param {object} propertyDescriptor The property descriptor
- */
- isDynamicProperty: function (propertyDescriptor) {
- if (nf.Common.isDefinedAndNotNull(propertyDescriptor)) {
- return propertyDescriptor.dynamic === true;
- } else {
- return false;
- }
- },
-
- /**
- * Gets the allowable values for the specified property.
- *
- * @argument {object} propertyDescriptor The property descriptor
- */
- getAllowableValues: function (propertyDescriptor) {
- if (nf.Common.isDefinedAndNotNull(propertyDescriptor)) {
- return propertyDescriptor.allowableValues;
- } else {
- return null;
- }
- },
-
- /**
- * Returns whether the specified property supports EL.
- *
- * @param {object} propertyDescriptor The property descriptor
- */
- supportsEl: function (propertyDescriptor) {
- if (nf.Common.isDefinedAndNotNull(propertyDescriptor)) {
- return propertyDescriptor.supportsEl === true;
- } else {
- return false;
- }
- },
-
- /**
- * Creates a form inline in order to submit the specified params to the specified URL
- * using the specified method.
- *
- * @param {string} url The URL
- * @param {object} params An object with the params to include in the submission
- */
- post: function (url, params) {
- // temporarily override beforeunload
- var previousBeforeUnload = window.onbeforeunload;
- window.onbeforeunload = null;
-
- // create a form for submission
- var form = $('<form></form>').attr({
- 'method': 'POST',
- 'action': url,
- 'style': 'display: none;'
- });
-
- // add each parameter when specified
- if (nf.Common.isDefinedAndNotNull(params)) {
- $.each(params, function (name, value) {
- $('<textarea></textarea>').attr('name', name).val(value).appendTo(form);
- });
- }
+ },
- // submit the form and clean up
- form.appendTo('body').submit().remove();
+ /**
+ * HTML escapes the specified string. If the string is null
+ * or undefined, an empty string is returned.
+ *
+ * @returns {string}
+ */
+ escapeHtml: (function () {
+ var entityMap = {
+ '&': '&',
+ '<': '<',
+ '>': '>',
+ '"': '"',
+ "'": ''',
+ '/': '/'
+ };
- // restore previous beforeunload if necessary
- if (previousBeforeUnload !== null) {
- window.onbeforeunload = previousBeforeUnload;
- }
- },
-
- /**
- * Formats the specified array as an unordered list. If the array is not an
- * array, null is returned.
- *
- * @argument {array} array The array to convert into an unordered list
- */
- formatUnorderedList: function (array) {
- if ($.isArray(array)) {
- var ul = $('<ul class="result"></ul>');
- $.each(array, function (_, item) {
- var li = $('<li></li>').appendTo(ul);
- if (item instanceof jQuery) {
- li.append(item);
+ return function (string) {
+ if (nf.Common.isDefinedAndNotNull(string)) {
+ return String(string).replace(/[&<>"'\/]/g, function (s) {
+ return entityMap[s];
+ });
} else {
- li.text(item);
+ return '';
}
+ };
+ }()),
+
+ /**
+ * Determines if the specified property is sensitive.
+ *
+ * @argument {object} propertyDescriptor The property descriptor
+ */
+ isSensitiveProperty: function (propertyDescriptor) {
+ if (nf.Common.isDefinedAndNotNull(propertyDescriptor)) {
+ return propertyDescriptor.sensitive === true;
+ } else {
+ return false;
+ }
+ },
+
+ /**
+ * Determines if the specified property is required.
+ *
+ * @param {object} propertyDescriptor The property descriptor
+ */
+ isRequiredProperty: function (propertyDescriptor) {
+ if (nf.Common.isDefinedAndNotNull(propertyDescriptor)) {
+ return propertyDescriptor.required === true;
+ } else {
+ return false;
+ }
+ },
+
+ /**
+ * Determines if the specified property is required.
+ *
+ * @param {object} propertyDescriptor The property descriptor
+ */
+ isDynamicProperty: function (propertyDescriptor) {
+ if (nf.Common.isDefinedAndNotNull(propertyDescriptor)) {
+ return propertyDescriptor.dynamic === true;
+ } else {
+ return false;
+ }
+ },
+
+ /**
+ * Gets the allowable values for the specified property.
+ *
+ * @argument {object} propertyDescriptor The property descriptor
+ */
+ getAllowableValues: function (propertyDescriptor) {
+ if (nf.Common.isDefinedAndNotNull(propertyDescriptor)) {
+ return propertyDescriptor.allowableValues;
+ } else {
+ return null;
+ }
+ },
+
+ /**
+ * Returns whether the specified property supports EL.
+ *
+ * @param {object} propertyDescriptor The property descriptor
+ */
+ supportsEl: function (propertyDescriptor) {
+ if (nf.Common.isDefinedAndNotNull(propertyDescriptor)) {
+ return propertyDescriptor.supportsEl === true;
+ } else {
+ return false;
+ }
+ },
+
+ /**
+ * Creates a form inline in order to submit the specified params to the specified URL
+ * using the specified method.
+ *
+ * @param {string} url The URL
+ * @param {object} params An object with the params to include in the submission
+ */
+ post: function (url, params) {
+ // temporarily override beforeunload
+ var previousBeforeUnload = window.onbeforeunload;
+ window.onbeforeunload = null;
+
+ // create a form for submission
+ var form = $('<form></form>').attr({
+ 'method': 'POST',
+ 'action': url,
+ 'style': 'display: none;'
});
- return ul;
- } else {
- return null;
- }
- },
-
- /**
- * Extracts the contents of the specified str after the strToFind. If the
- * strToFind is not found or the last part of the str, an empty string is
- * returned.
- *
- * @argument {string} str The full string
- * @argument {string} strToFind The substring to find
- */
- substringAfterLast: function (str, strToFind) {
- var result = '';
- var indexOfStrToFind = str.lastIndexOf(strToFind);
- if (indexOfStrToFind >= 0) {
- var indexAfterStrToFind = indexOfStrToFind + strToFind.length;
- if (indexAfterStrToFind < str.length) {
- result = str.substr(indexAfterStrToFind);
+
+ // add each parameter when specified
+ if (nf.Common.isDefinedAndNotNull(params)) {
+ $.each(params, function (name, value) {
+ $('<textarea></textarea>').attr('name', name).val(value).appendTo(form);
+ });
}
- }
- return result;
- },
-
- /**
- * Updates the mouse pointer.
- *
- * @argument {string} domId The id of the element for the new cursor style
- * @argument {boolean} isMouseOver Whether or not the mouse is over the element
- */
- setCursor: function (domId, isMouseOver) {
- if (isMouseOver) {
- $('#' + domId).addClass('pointer');
- } else {
- $('#' + domId).removeClass('pointer');
- }
- },
-
- /**
- * Constants for time duration formatting.
- */
- MILLIS_PER_DAY: 86400000,
- MILLIS_PER_HOUR: 3600000,
- MILLIS_PER_MINUTE: 60000,
- MILLIS_PER_SECOND: 1000,
-
- /**
- * Formats the specified duration.
- *
- * @param {integer} duration in millis
- */
- formatDuration: function (duration) {
- // don't support sub millisecond resolution
- duration = duration < 1 ? 0 : duration;
-
- // determine the number of days in the specified duration
- var days = duration / nf.Common.MILLIS_PER_DAY;
- days = days >= 1 ? parseInt(days, 10) : 0;
- duration %= nf.Common.MILLIS_PER_DAY;
-
- // remaining duration should be less than 1 day, get number of hours
- var hours = duration / nf.Common.MILLIS_PER_HOUR;
- hours = hours >= 1 ? parseInt(hours, 10) : 0;
- duration %= nf.Common.MILLIS_PER_HOUR;
-
- // remaining duration should be less than 1 hour, get number of minutes
- var minutes = duration / nf.Common.MILLIS_PER_MINUTE;
- minutes = minutes >= 1 ? parseInt(minutes, 10) : 0;
- duration %= nf.Common.MILLIS_PER_MINUTE;
-
- // remaining duration should be less than 1 minute, get number of seconds
- var seconds = duration / nf.Common.MILLIS_PER_SECOND;
- seconds = seconds >= 1 ? parseInt(seconds, 10) : 0;
-
- // remaining duration is the number millis (don't support sub millisecond resolution)
- duration = Math.floor(duration % nf.Common.MILLIS_PER_SECOND);
-
- // format the time
- var time = nf.Common.pad(hours, 2, '0') +
- ':' +
- nf.Common.pad(minutes, 2, '0') +
- ':' +
- nf.Common.pad(seconds, 2, '0') +
- '.' +
- nf.Common.pad(duration, 3, '0');
-
- // only include days if appropriate
- if (days > 0) {
- return days + ' days and ' + time;
- } else {
- return time;
- }
- },
-
- /**
- * Constants for formatting data size.
- */
- BYTES_IN_KILOBYTE: 1024,
- BYTES_IN_MEGABYTE: 1048576,
- BYTES_IN_GIGABYTE: 1073741824,
- BYTES_IN_TERABYTE: 1099511627776,
-
- /**
- * Formats the specified number of bytes into a human readable string.
- *
- * @param {integer} dataSize
- * @returns {string}
- */
- formatDataSize: function (dataSize) {
- // check terabytes
- var dataSizeToFormat = parseFloat(dataSize / nf.Common.BYTES_IN_TERABYTE);
- if (dataSizeToFormat > 1) {
- return dataSizeToFormat.toFixed(2) + " TB";
- }
- // check gigabytes
- dataSizeToFormat = parseFloat(dataSize / nf.Common.BYTES_IN_GIGABYTE);
- if (dataSizeToFormat > 1) {
- return dataSizeToFormat.toFixed(2) + " GB";
- }
+ // submit the form and clean up
+ form.appendTo('body').submit().remove();
- // check megabytes
- dataSizeToFormat = parseFloat(dataSize / nf.Common.BYTES_IN_MEGABYTE);
- if (dataSizeToFormat > 1) {
- return dataSizeToFormat.toFixed(2) + " MB";
- }
+ // restore previous beforeunload if necessary
+ if (previousBeforeUnload !== null) {
+ window.onbeforeunload = previousBeforeUnload;
+ }
+ },
- // check kilobytes
- dataSizeToFormat = parseFloat(dataSize / nf.Common.BYTES_IN_KILOBYTE);
- if (dataSizeToFormat > 1) {
- return dataSizeToFormat.toFixed(2) + " KB";
- }
+ /**
+ * Formats the specified array as an unordered list. If the array is not an
+ * array, null is returned.
+ *
+ * @argument {array} array The array to convert into an unordered list
+ */
+ formatUnorderedList: function (array) {
+ if ($.isArray(array)) {
+ var ul = $('<ul class="result"></ul>');
+ $.each(array, function (_, item) {
+ var li = $('<li></li>').appendTo(ul);
+ if (item instanceof jQuery) {
+ li.append(item);
+ } else {
+ li.text(item);
+ }
+ });
+ return ul;
+ } else {
+ return null;
+ }
+ },
- // default to bytes
- return parseFloat(dataSize).toFixed(2) + " bytes";
- },
-
- /**
- * Formats the specified integer as a string (adding commas). At this
- * point this does not take into account any locales.
- *
- * @param {integer} integer
- */
- formatInteger: function (integer) {
- var string = integer + '';
- var regex = /(\d+)(\d{3})/;
- while (regex.test(string)) {
- string = string.replace(regex, '$1' + ',' + '$2');
- }
- return string;
- },
-
- /**
- * Formats the specified float using two demical places.
- *
- * @param {float} f
- */
- formatFloat: function (f) {
- if (nf.Common.isUndefinedOrNull(f)) {
- return 0.00 + '';
- }
- return f.toFixed(2) + '';
- },
-
- /**
- * Pads the specified value to the specified width with the specified character.
- * If the specified value is already wider than the specified width, the original
- * value is returned.
- *
- * @param {integer} value
- * @param {integer} width
- * @param {string} character
- * @returns {string}
- */
- pad: function (value, width, character) {
- var s = value + '';
-
- // pad until wide enough
- while (s.length < width) {
- s = character + s;
- }
+ /**
+ * Extracts the contents of the specified str after the strToFind. If the
+ * strToFind is not found or the last part of the str, an empty string is
+ * returned.
+ *
+ * @argument {string} str The full string
+ * @argument {string} strToFind The substring to find
+ */
+ substringAfterLast: function (str, strToFind) {
+ var result = '';
+ var indexOfStrToFind = str.lastIndexOf(strToFind);
+ if (indexOfStrToFind >= 0) {
+ var indexAfterStrToFind = indexOfStrToFind + strToFind.length;
+ if (indexAfterStrToFind < str.length) {
+ result = str.substr(indexAfterStrToFind);
+ }
+ }
+ return result;
+ },
- return s;
- },
-
- /**
- * Formats the specified DateTime.
- *
- * @param {Date} date
- * @returns {String}
- */
- formatDateTime: function (date) {
- return nf.Common.pad(date.getMonth() + 1, 2, '0') +
- '/' +
- nf.Common.pad(date.getDate(), 2, '0') +
- '/' +
- nf.Common.pad(date.getFullYear(), 2, '0') +
- ' ' +
- nf.Common.pad(date.getHours(), 2, '0') +
- ':' +
- nf.Common.pad(date.getMinutes(), 2, '0') +
- ':' +
- nf.Common.pad(date.getSeconds(), 2, '0') +
- '.' +
- nf.Common.pad(date.getMilliseconds(), 3, '0');
- },
-
- /**
- * Parses the specified date time into a Date object. The resulting
- * object does not account for timezone and should only be used for
- * performing relative comparisons.
- *
- * @param {string} rawDateTime
- * @returns {Date}
- */
- parseDateTime: function (rawDateTime) {
- // handle non date values
- if (!nf.Common.isDefinedAndNotNull(rawDateTime)) {
- return new Date();
- }
- if (rawDateTime === 'No value set') {
- return new Date();
- }
- if (rawDateTime === 'Empty string set') {
- return new Date();
- }
+ /**
+ * Updates the mouse pointer.
+ *
+ * @argument {string} domId The id of the element for the new cursor style
+ * @argument {boolean} isMouseOver Whether or not the mouse is over the element
+ */
+ setCursor: function (domId, isMouseOver) {
+ if (isMouseOver) {
+ $('#' + domId).addClass('pointer');
+ } else {
+ $('#' + domId).removeClass('pointer');
+ }
+ },
- // parse the date time
- var dateTime = rawDateTime.split(/ /);
+ /**
+ * Constants for time duration formatting.
+ */
+ MILLIS_PER_DAY: 86400000,
+ MILLIS_PER_HOUR: 3600000,
+ MILLIS_PER_MINUTE: 60000,
+ MILLIS_PER_SECOND: 1000,
- // ensure the correct number of tokens
- if (dateTime.length !== 3) {
- return new Date();
- }
+ /**
+ * Formats the specified duration.
+ *
+ * @param {integer} duration in millis
+ */
+ formatDuration: function (duration) {
+ // don't support sub millisecond resolution
+ duration = duration < 1 ? 0 : duration;
- // get the date and time
- var date = dateTime[0].split(/\//);
- var time = dateTime[1].split(/:/);
+ // determine the number of days in the specified duration
+ var days = duration / nf.Common.MILLIS_PER_DAY;
+ days = days >= 1 ? parseInt(days, 10) : 0;
+ duration %= nf.Common.MILLIS_PER_DAY;
- // ensure the correct number of tokens
- if (date.length !== 3 || time.length !== 3) {
- return new Date();
- }
+ // remaining duration should be less than 1 day, get number of hours
+ var hours = duration / nf.Common.MILLIS_PER_HOUR;
+ hours = hours >= 1 ? parseInt(hours, 10) : 0;
+ duration %= nf.Common.MILLIS_PER_HOUR;
- // detect if there is millis
- var seconds = time[2].split(/\./);
- if (seconds.length === 2) {
- return new Date(parseInt(date[2], 10), parseInt(date[0], 10), parseInt(date[1], 10), parseInt(time[0], 10), parseInt(time[1], 10), parseInt(seconds[0], 10), parseInt(seconds[1], 10));
- } else {
- return new Date(parseInt(date[2], 10), parseInt(date[0], 10), parseInt(date[1], 10), parseInt(time[0], 10), parseInt(time[1], 10), parseInt(time[2], 10), 0);
- }
- },
-
- /**
- * Parses the specified duration and returns the total number of millis.
- *
- * @param {string} rawDuration
- * @returns {number} The number of millis
- */
- parseDuration: function (rawDuration) {
- var duration = rawDuration.split(/:/);
-
- // ensure the appropriate number of tokens
- if (duration.length !== 3) {
- return 0;
- }
+ // remaining duration should be less than 1 hour, get number of minutes
+ var minutes = duration / nf.Common.MILLIS_PER_MINUTE;
+ minutes = minutes >= 1 ? parseInt(minutes, 10) : 0;
+ duration %= nf.Common.MILLIS_PER_MINUTE;
- // detect if there is millis
- var seconds = duration[2].split(/\./);
- if (seconds.length === 2) {
- return new Date(1970, 0, 1, parseInt(duration[0], 10), parseInt(duration[1], 10), parseInt(seconds[0], 10), parseInt(seconds[1], 10)).getTime();
- } else {
- return new Date(1970, 0, 1, parseInt(duration[0], 10), parseInt(duration[1], 10), parseInt(duration[2], 10), 0).getTime();
- }
- },
-
- /**
- * Parses the specified size.
- *
- * @param {string} rawSize
- * @returns {int}
- */
- parseSize: function (rawSize) {
- var tokens = rawSize.split(/ /);
- var size = parseFloat(tokens[0].replace(/,/g, ''));
- var units = tokens[1];
-
- if (units === 'KB') {
- return size * 1024;
- } else if (units === 'MB') {
- return size * 1024 * 1024;
- } else if (units === 'GB') {
- return size * 1024 * 1024 * 1024;
- } else if (units === 'TB') {
- return size * 1024 * 1024 * 1024 * 1024;
- } else {
- return size;
- }
- },
-
- /**
- * Parses the specified count.
- *
- * @param {string} rawCount
- * @returns {int}
- */
- parseCount: function (rawCount) {
- // extract the count
- var count = rawCount.split(/ /, 1);
-
- // ensure the string was split successfully
- if (count.length !== 1) {
- return 0;
- }
+ // remaining duration should be less than 1 minute, get number of seconds
+ var seconds = duration / nf.Common.MILLIS_PER_SECOND;
+ seconds = seconds >= 1 ? parseInt(seconds, 10) : 0;
- // convert the count to an integer
- var intCount = parseInt(count[0].replace(/,/g, ''), 10);
+ // remaining duration is the number millis (don't support sub millisecond resolution)
+ duration = Math.floor(duration % nf.Common.MILLIS_PER_SECOND);
- // ensure it was parsable as an integer
- if (isNaN(intCount)) {
- return 0;
- }
- return intCount;
- },
-
- /**
- * Determines if the specified object is defined and not null.
- *
- * @argument {object} obj The object to test
- */
- isDefinedAndNotNull: function (obj) {
- return !nf.Common.isUndefined(obj) && !nf.Common.isNull(obj);
- },
-
- /**
- * Determines if the specified object is undefined or null.
- *
- * @param {object} obj The object to test
- */
- isUndefinedOrNull: function (obj) {
- return nf.Common.isUndefined(obj) || nf.Common.isNull(obj);
- },
-
- /**
- * Determines if the specified object is undefined.
- *
- * @argument {object} obj The object to test
- */
- isUndefined: function (obj) {
- return typeof obj === 'undefined';
- },
-
- /**
- * Determines whether the specified string is blank (or null or undefined).
- *
- * @argument {string} str The string to test
- */
- isBlank: function (str) {
- return nf.Common.isUndefined(str) || nf.Common.isNull(str) || $.trim(str) === '';
- },
-
- /**
- * Determines if the specified object is null.
- *
- * @argument {object} obj The object to test
- */
- isNull: function (obj) {
- return obj === null;
- },
-
- /**
- * Determines if the specified array is empty. If the specified arg is not an
- * array, then true is returned.
- *
- * @argument {array} arr The array to test
- */
- isEmpty: function (arr) {
- return $.isArray(arr) ? arr.length === 0 : true;
- },
-
- /**
- * Determines if these are the same bulletins. If both arguments are not
- * arrays, false is returned.
- *
- * @param {array} bulletins
- * @param {array} otherBulletins
- * @returns {boolean}
- */
- doBulletinsDiffer: function (bulletins, otherBulletins) {
- if ($.isArray(bulletins) && $.isArray(otherBulletins)) {
- if (bulletins.length === otherBulletins.length) {
- for (var i = 0; i < bulletins.length; i++) {
- if (bulletins[i].id !== otherBulletins[i].id) {
- return true;
+ // format the time
+ var time = nf.Common.pad(hours, 2, '0') +
+ ':' +
+ nf.Common.pad(minutes, 2, '0') +
+ ':' +
+ nf.Common.pad(seconds, 2, '0') +
+ '.' +
+ nf.Common.pad(duration, 3, '0');
+
+ // only include days if appropriate
+ if (days > 0) {
+ return days + ' days and ' + time;
+ } else {
+ return time;
+ }
+ },
+
+ /**
+ * Constants for formatting data size.
+ */
+ BYTES_IN_KILOBYTE: 1024,
+ BYTES_IN_MEGABYTE: 1048576,
+ BYTES_IN_GIGABYTE: 1073741824,
+ BYTES_IN_TERABYTE: 1099511627776,
+
+ /**
+ * Formats the specified number of bytes into a human readable string.
+ *
+ * @param {integer} dataSize
+ * @returns {string}
+ */
+ formatDataSize: function (dataSize) {
+ // check terabytes
+ var dataSizeToFormat = parseFloat(dataSize / nf.Common.BYTES_IN_TERABYTE);
+ if (dataSizeToFormat > 1) {
+ return dataSizeToFormat.toFixed(2) + " TB";
+ }
+
+ // check gigabytes
+ dataSizeToFormat = parseFloat(dataSize / nf.Common.BYTES_IN_GIGABYTE);
+ if (dataSizeToFormat > 1) {
+ return dataSizeToFormat.toFixed(2) + " GB";
+ }
+
+ // check megabytes
+ dataSizeToFormat = parseFloat(dataSize / nf.Common.BYTES_IN_MEGABYTE);
+ if (dataSizeToFormat > 1) {
+ return dataSizeToFormat.toFixed(2) + " MB";
+ }
+
+ // check kilobytes
+ dataSizeToFormat = parseFloat(dataSize / nf.Common.BYTES_IN_KILOBYTE);
+ if (dataSizeToFormat > 1) {
+ return dataSizeToFormat.toFixed(2) + " KB";
+ }
+
+ // default to bytes
+ return parseFloat(dataSize).toFixed(2) + " bytes";
+ },
+
+ /**
+ * Formats the specified integer as a string (adding commas). At this
+ * point this does not take into account any locales.
+ *
+ * @param {integer} integer
+ */
+ formatInteger: function (integer) {
+ var string = integer + '';
+ var regex = /(\d+)(\d{3})/;
+ while (regex.test(string)) {
+ string = string.replace(regex, '$1' + ',' + '$2');
+ }
+ return string;
+ },
+
+ /**
+ * Formats the specified float using two demical places.
+ *
+ * @param {float} f
+ */
+ formatFloat: function (f) {
+ if (nf.Common.isUndefinedOrNull(f)) {
+ return 0.00 + '';
+ }
+ return f.toFixed(2) + '';
+ },
+
+ /**
+ * Pads the specified value to the specified width with the specified character.
+ * If the specified value is already wider than the specified width, the original
+ * value is returned.
+ *
+ * @param {integer} value
+ * @param {integer} width
+ * @param {string} character
+ * @returns {string}
+ */
+ pad: function (value, width, character) {
+ var s = value + '';
+
+ // pad until wide enough
+ while (s.length < width) {
+ s = character + s;
+ }
+
+ return s;
+ },
+
+ /**
+ * Formats the specified DateTime.
+ *
+ * @param {Date} date
+ * @returns {String}
+ */
+ formatDateTime: function (date) {
+ return nf.Common.pad(date.getMonth() + 1, 2, '0') +
+ '/' +
+ nf.Common.pad(date.getDate(), 2, '0') +
+ '/' +
+ nf.Common.pad(date.getFullYear(), 2, '0') +
+ ' ' +
+ nf.Common.pad(date.getHours(), 2, '0') +
+ ':' +
+ nf.Common.pad(date.getMinutes(), 2, '0') +
+ ':' +
+ nf.Common.pad(date.getSeconds(), 2, '0') +
+ '.' +
+ nf.Common.pad(date.getMilliseconds(), 3, '0');
+ },
+
+ /**
+ * Parses the specified date time into a Date object. The resulting
+ * object does not account for timezone and should only be used for
+ * performing relative comparisons.
+ *
+ * @param {string} rawDateTime
+ * @returns {Date}
+ */
+ parseDateTime: function (rawDateTime) {
+ // handle non date values
+ if (!nf.Common.isDefinedAndNotNull(rawDateTime)) {
+ return new Date();
+ }
+ if (rawDateTime === 'No value set') {
+ return new Date();
+ }
+ if (rawDateTime === 'Empty string set') {
+ return new Date();
+ }
+
+ // parse the date time
+ var dateTime = rawDateTime.split(/ /);
+
+ // ensure the correct number of tokens
+ if (dateTime.length !== 3) {
+ return new Date();
+ }
+
+ // get the date and time
+ var date = dateTime[0].split(/\//);
+ var time = dateTime[1].split(/:/);
+
+ // ensure the correct number of tokens
+ if (date.length !== 3 || time.length !== 3) {
+ return new Date();
+ }
+
+ // detect if there is millis
+ var seconds = time[2].split(/\./);
+ if (seconds.length === 2) {
+ return new Date(parseInt(date[2], 10), parseInt(date[0], 10), parseInt(date[1], 10), parseInt(time[0], 10), parseInt(time[1], 10), parseInt(seconds[0], 10), parseInt(seconds[1], 10));
+ } else {
+ return new Date(parseInt(date[2], 10), parseInt(date[0], 10), parseInt(date[1], 10), parseInt(time[0], 10), parseInt(time[1], 10), parseInt(time[2], 10), 0);
+ }
+ },
+
+ /**
+ * Parses the specified duration and returns the total number of millis.
+ *
+ * @param {string} rawDuration
+ * @returns {number} The number of millis
+ */
+ parseDuration: function (rawDuration) {
+ var duration = rawDuration.split(/:/);
+
+ // ensure the appropriate number of tokens
+ if (duration.length !== 3) {
+ return 0;
+ }
+
+ // detect if there is millis
+ var seconds = duration[2].split(/\./);
+ if (seconds.length === 2) {
+ return new Date(1970, 0, 1, parseInt(duration[0], 10), parseInt(duration[1], 10), parseInt(seconds[0], 10), parseInt(seconds[1], 10)).getTime();
+ } else {
+ return new Date(1970, 0, 1, parseInt(duration[0], 10), parseInt(duration[1], 10), parseInt(duration[2], 10), 0).getTime();
+ }
+ },
+
+ /**
+ * Parses the specified size.
+ *
+ * @param {string} rawSize
+ * @returns {int}
+ */
+ parseSize: function (rawSize) {
+ var tokens = rawSize.split(/ /);
+ var size = parseFloat(tokens[0].replace(/,/g, ''));
+ var units = tokens[1];
+
+ if (units === 'KB') {
+ return size * 1024;
+ } else if (units === 'MB') {
+ return size * 1024 * 1024;
+ } else if (units === 'GB') {
+ return size * 1024 * 1024 * 1024;
+ } else if (units === 'TB') {
+ return size * 1024 * 1024 * 1024 * 1024;
+ } else {
+ return size;
+ }
+ },
+
+ /**
+ * Parses the specified count.
+ *
+ * @param {string} rawCount
+ * @returns {int}
+ */
+ parseCount: function (rawCount) {
+ // extract the count
+ var count = rawCount.split(/ /, 1);
+
+ // ensure the string was split successfully
+ if (count.length !== 1) {
+ return 0;
+ }
+
+ // convert the count to an integer
+ var intCount = parseInt(count[0].replace(/,/g, ''), 10);
+
+ // ensure it was parsable as an integer
+ if (isNaN(intCount)) {
+ return 0;
+ }
+ return intCount;
+ },
+
+ /**
+ * Determines if the specified object is defined and not null.
+ *
+ * @argument {object} obj The object to test
+ */
+ isDefinedAndNotNull: function (obj) {
+ return !nf.Common.isUndefined(obj) && !nf.Common.isNull(obj);
+ },
+
+ /**
+ * Determines if the specified object is undefined or null.
+ *
+ * @param {object} obj The object to test
+ */
+ isUndefinedOrNull: function (obj) {
+ return nf.Common.isUndefined(obj) || nf.Common.isNull(obj);
+ },
+
+ /**
+ * Determines if the specified object is undefined.
+ *
+ * @argument {object} obj The object to test
+ */
+ isUndefined: function (obj) {
+ return typeof obj === 'undefined';
+ },
+
+ /**
+ * Determines whether the specified string is blank (or null or undefined).
+ *
+ * @argument {string} str The string to test
+ */
+ isBlank: function (str) {
+ return nf.Common.isUndefined(str) || nf.Common.isNull(str) || $.trim(str) === '';
+ },
+
+ /**
+ * Determines if the specified object is null.
+ *
+ * @argument {object} obj The object to test
+ */
+ isNull: function (obj) {
+ return obj === null;
+ },
+
+ /**
+ * Determines if the specified array is empty. If the specified arg is not an
+ * array, then true is returned.
+ *
+ * @argument {array} arr The array to test
+ */
+ isEmpty: function (arr) {
+ return $.isArray(arr) ? arr.length === 0 : true;
+ },
+
+ /**
+ * Determines if these are the same bulletins. If both arguments are not
+ * arrays, false is returned.
+ *
+ * @param {array} bulletins
+ * @param {array} otherBulletins
+ * @returns {boolean}
+ */
+ doBulletinsDiffer: function (bulletins, otherBulletins) {
+ if ($.isArray(bulletins) && $.isArray(otherBulletins)) {
+ if (bulletins.length === otherBulletins.length) {
+ for (var i = 0; i < bulletins.length; i++) {
+ if (bulletins[i].id !== otherBulletins[i].id) {
+ return true;
+ }
}
+ } else {
+ return true;
}
- } else {
+ } else if ($.isArray(bulletins) || $.isArray(otherBulletins)) {
return true;
}
- } else if ($.isArray(bulletins) || $.isArray(otherBulletins)) {
- return true;
+ return false;
+ },
+
+ /**
+ * Formats the specified bulletin list.
+ *
+ * @argument {array} bulletins The bulletins
+ * @return {array} The jQuery objects
+ */
+ getFormattedBulletins: function (bulletins) {
+ var formattedBulletins = [];
+ $.each(bulletins, function (j, bulletin) {
+ // format the node address
+ var nodeAddress = '';
+ if (nf.Common.isDefinedAndNotNull(bulletin.nodeAddress)) {
+ nodeAddress = '- ' + nf.Common.escapeHtml(bulletin.nodeAddress) + ' - ';
+ }
+
+ // set the bulletin message (treat as text)
+ var bulletinMessage = $('<pre></pre>').css({
+ 'white-space': 'pre-wrap'
+ }).text(bulletin.message);
+
+ // create the bulletin message
+ var formattedBulletin = $('<div>' +
+ nf.Common.escapeHtml(bulletin.timestamp) + ' ' +
+ nodeAddress + ' ' +
+ '<b>' + nf.Common.escapeHtml(bulletin.level) + '</b> ' +
+ '</div>').append(bulletinMessage);
+
+ formattedBulletins.push(formattedBulletin);
+ });
+ return formattedBulletins;
}
- return false;
- },
-
- /**
- * Formats the specified bulletin list.
- *
- * @argument {array} bulletins The bulletins
- * @return {array} The jQuery objects
- */
- getFormattedBulletins: function (bulletins) {
- var formattedBulletins = [];
- $.each(bulletins, function (j, bulletin) {
- // format the node address
- var nodeAddress = '';
- if (nf.Common.isDefinedAndNotNull(bulletin.nodeAddress)) {
- nodeAddress = '- ' + nf.Common.escapeHtml(bulletin.nodeAddress) + ' - ';
- }
-
- // set the bulletin message (treat as text)
- var bulletinMessage = $('<pre></pre>').css({
- 'white-space': 'pre-wrap'
- }).text(bulletin.message);
-
- // create the bulletin message
- var formattedBulletin = $('<div>' +
- nf.Common.escapeHtml(bulletin.timestamp) + ' ' +
- nodeAddress + ' ' +
- '<b>' + nf.Common.escapeHtml(bulletin.level) + '</b> ' +
- '</div>').append(bulletinMessage);
-
- formattedBulletins.push(formattedBulletin);
- });
- return formattedBulletins;
- }
-};
+ };
+}());
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/nf-dialog.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/nf-dialog.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/nf-dialog.js
index 13f5cf3..9a926ef 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/nf-dialog.js
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/nf-dialog.js
@@ -23,15 +23,6 @@ $(document).ready(function () {
// configure the ok dialog
$('#nf-ok-dialog').modal({
- buttons: [{
- buttonText: 'Ok',
- handler: {
- click: function () {
- // close the dialog
- $('#nf-ok-dialog').modal('hide');
- }
- }
- }],
handler: {
close: function () {
// clear the content
@@ -78,6 +69,20 @@ nf.Dialog = (function () {
var content = $('<p></p>').append(options.dialogContent);
$('#nf-ok-dialog-content').append(content);
+ // update the button model
+ $('#nf-ok-dialog').modal('setButtonModel', [{
+ buttonText: 'Ok',
+ handler: {
+ click: function () {
+ // close the dialog
+ $('#nf-ok-dialog').modal('hide');
+ if (typeof options.okHandler === 'function') {
+ options.okHandler.call(this);
+ }
+ }
+ }
+ }]);
+
// show the dialog
$('#nf-ok-dialog').modal('setHeaderText', options.headerText).modal('setOverlayBackground', options.overlayBackground).modal('show');
},
[45/50] [abbrv] nifi git commit: NIFI-655: - Refactoring web security
to use Spring Security Java Configuration. - Introducing security in Web UI
in order to get JWT.
Posted by mc...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/integration/util/NiFiTestUser.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/integration/util/NiFiTestUser.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/integration/util/NiFiTestUser.java
index 52f4522..621dc09 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/integration/util/NiFiTestUser.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/integration/util/NiFiTestUser.java
@@ -22,8 +22,7 @@ import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.core.util.MultivaluedMapImpl;
import java.util.Map;
import javax.ws.rs.core.MediaType;
-import org.apache.nifi.web.security.DnUtils;
-import org.apache.nifi.web.security.x509.X509AuthenticationFilter;
+import org.apache.nifi.web.security.ProxiedEntitiesUtils;
/**
*
@@ -35,9 +34,27 @@ public class NiFiTestUser {
private final Client client;
private final String proxyDn;
- public NiFiTestUser(Client client, String dn) {
+ public NiFiTestUser(Client client, String proxyDn) {
this.client = client;
- this.proxyDn = DnUtils.formatProxyDn(dn);
+ if (proxyDn != null) {
+ this.proxyDn = ProxiedEntitiesUtils.formatProxyDn(proxyDn);
+ } else {
+ this.proxyDn = null;
+ }
+ }
+
+ /**
+ * Conditionally adds the proxied entities chain.
+ *
+ * @param builder the resource builder
+ * @return the resource builder
+ */
+ private WebResource.Builder addProxiedEntities(final WebResource.Builder builder) {
+ if (proxyDn == null) {
+ return builder;
+ } else {
+ return builder.header(ProxiedEntitiesUtils.PROXY_ENTITIES_CHAIN, proxyDn);
+ }
}
/**
@@ -59,6 +76,18 @@ public class NiFiTestUser {
* @return response
*/
public ClientResponse testGet(String url, Map<String, String> queryParams) {
+ return testGetWithHeaders(url, queryParams, null);
+ }
+
+ /**
+ * Performs a GET using the specified url and query parameters.
+ *
+ * @param url url
+ * @param queryParams params
+ * @param headers http headers
+ * @return response
+ */
+ public ClientResponse testGetWithHeaders(String url, Map<String, String> queryParams, Map<String, String> headers) {
// get the resource
WebResource resource = client.resource(url);
@@ -69,8 +98,18 @@ public class NiFiTestUser {
}
}
+ // get the builder
+ WebResource.Builder builder = addProxiedEntities(resource.accept(MediaType.APPLICATION_JSON));
+
+ // append any headers
+ if (headers != null && !headers.isEmpty()) {
+ for (String key : headers.keySet()) {
+ builder = builder.header(key, headers.get(key));
+ }
+ }
+
// perform the query
- return resource.accept(MediaType.APPLICATION_JSON).header(X509AuthenticationFilter.PROXY_ENTITIES_CHAIN, proxyDn).get(ClientResponse.class);
+ return builder.get(ClientResponse.class);
}
/**
@@ -93,14 +132,34 @@ public class NiFiTestUser {
* @throws Exception ex
*/
public ClientResponse testPost(String url, Object entity) throws Exception {
+ return testPostWithHeaders(url, entity, null);
+ }
+
+ /**
+ * Performs a POST using the specified url and entity body.
+ *
+ * @param url url
+ * @param entity entity
+ * @param headers http headers
+ * @return response
+ * @throws Exception ex
+ */
+ public ClientResponse testPostWithHeaders(String url, Object entity, Map<String, String> headers) throws Exception {
// get the resource
- WebResource.Builder resourceBuilder = client.resource(url).accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON).header(X509AuthenticationFilter.PROXY_ENTITIES_CHAIN, proxyDn);
+ WebResource.Builder resourceBuilder = addProxiedEntities(client.resource(url).accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON));
// include the request entity
if (entity != null) {
resourceBuilder = resourceBuilder.entity(entity);
}
+ // append any headers
+ if (headers != null && !headers.isEmpty()) {
+ for (String key : headers.keySet()) {
+ resourceBuilder = resourceBuilder.header(key, headers.get(key));
+ }
+ }
+
// perform the request
return resourceBuilder.post(ClientResponse.class);
}
@@ -110,18 +169,38 @@ public class NiFiTestUser {
*
* @param url url
* @param entity entity
- * @return repsonse
+ * @return response
* @throws Exception ex
*/
public ClientResponse testPostMultiPart(String url, Object entity) throws Exception {
+ return testPostMultiPartWithHeaders(url, entity, null);
+ }
+
+ /**
+ * Performs a POST using the specified url and entity body.
+ *
+ * @param url url
+ * @param entity entity
+ * @param headers http headers
+ * @return response
+ * @throws Exception ex
+ */
+ public ClientResponse testPostMultiPartWithHeaders(String url, Object entity, Map<String, String> headers) throws Exception {
// get the resource
- WebResource.Builder resourceBuilder = client.resource(url).accept(MediaType.APPLICATION_XML).type(MediaType.MULTIPART_FORM_DATA).header(X509AuthenticationFilter.PROXY_ENTITIES_CHAIN, proxyDn);
+ WebResource.Builder resourceBuilder = addProxiedEntities(client.resource(url).accept(MediaType.APPLICATION_XML).type(MediaType.MULTIPART_FORM_DATA));
// include the request entity
if (entity != null) {
resourceBuilder = resourceBuilder.entity(entity);
}
+ // append any headers
+ if (headers != null && !headers.isEmpty()) {
+ for (String key : headers.keySet()) {
+ resourceBuilder = resourceBuilder.header(key, headers.get(key));
+ }
+ }
+
// perform the request
return resourceBuilder.post(ClientResponse.class);
}
@@ -135,6 +214,19 @@ public class NiFiTestUser {
* @throws java.lang.Exception ex
*/
public ClientResponse testPost(String url, Map<String, String> formData) throws Exception {
+ return testPostWithHeaders(url, formData, null);
+ }
+
+ /**
+ * Performs a POST using the specified url and form data.
+ *
+ * @param url url
+ * @param formData form data
+ * @param headers http headers
+ * @return response
+ * @throws java.lang.Exception ex
+ */
+ public ClientResponse testPostWithHeaders(String url, Map<String, String> formData, Map<String, String> headers) throws Exception {
// convert the form data
MultivaluedMapImpl entity = new MultivaluedMapImpl();
for (String key : formData.keySet()) {
@@ -142,14 +234,20 @@ public class NiFiTestUser {
}
// get the resource
- WebResource.Builder resourceBuilder
- = client.resource(url).accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_FORM_URLENCODED).header(X509AuthenticationFilter.PROXY_ENTITIES_CHAIN, proxyDn);
+ WebResource.Builder resourceBuilder = addProxiedEntities(client.resource(url).accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_FORM_URLENCODED));
// add the form data if necessary
if (!entity.isEmpty()) {
resourceBuilder = resourceBuilder.entity(entity);
}
+ // append any headers
+ if (headers != null && !headers.isEmpty()) {
+ for (String key : headers.keySet()) {
+ resourceBuilder = resourceBuilder.header(key, headers.get(key));
+ }
+ }
+
// perform the request
return resourceBuilder.post(ClientResponse.class);
}
@@ -163,14 +261,34 @@ public class NiFiTestUser {
* @throws java.lang.Exception ex
*/
public ClientResponse testPut(String url, Object entity) throws Exception {
+ return testPutWithHeaders(url, entity, null);
+ }
+
+ /**
+ * Performs a PUT using the specified url and entity body.
+ *
+ * @param url url
+ * @param entity entity
+ * @param headers http headers
+ * @return response
+ * @throws java.lang.Exception ex
+ */
+ public ClientResponse testPutWithHeaders(String url, Object entity, Map<String, String> headers) throws Exception {
// get the resource
- WebResource.Builder resourceBuilder = client.resource(url).accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON).header(X509AuthenticationFilter.PROXY_ENTITIES_CHAIN, proxyDn);
+ WebResource.Builder resourceBuilder = addProxiedEntities(client.resource(url).accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON));
// include the request entity
if (entity != null) {
resourceBuilder = resourceBuilder.entity(entity);
}
+ // append any headers
+ if (headers != null && !headers.isEmpty()) {
+ for (String key : headers.keySet()) {
+ resourceBuilder = resourceBuilder.header(key, headers.get(key));
+ }
+ }
+
// perform the request
return resourceBuilder.put(ClientResponse.class);
}
@@ -184,6 +302,19 @@ public class NiFiTestUser {
* @throws java.lang.Exception ex
*/
public ClientResponse testPut(String url, Map<String, String> formData) throws Exception {
+ return testPutWithHeaders(url, formData, null);
+ }
+
+ /**
+ * Performs a PUT using the specified url and form data.
+ *
+ * @param url url
+ * @param formData form data
+ * @param headers http headers
+ * @return response
+ * @throws java.lang.Exception ex
+ */
+ public ClientResponse testPutWithHeaders(String url, Map<String, String> formData, Map<String, String> headers) throws Exception {
// convert the form data
MultivaluedMapImpl entity = new MultivaluedMapImpl();
for (String key : formData.keySet()) {
@@ -191,14 +322,20 @@ public class NiFiTestUser {
}
// get the resource
- WebResource.Builder resourceBuilder
- = client.resource(url).accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_FORM_URLENCODED).header(X509AuthenticationFilter.PROXY_ENTITIES_CHAIN, proxyDn);
+ WebResource.Builder resourceBuilder = addProxiedEntities(client.resource(url).accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_FORM_URLENCODED));
// add the form data if necessary
if (!entity.isEmpty()) {
resourceBuilder = resourceBuilder.entity(entity);
}
+ // append any headers
+ if (headers != null && !headers.isEmpty()) {
+ for (String key : headers.keySet()) {
+ resourceBuilder = resourceBuilder.header(key, headers.get(key));
+ }
+ }
+
// perform the request
return resourceBuilder.put(ClientResponse.class);
}
@@ -211,24 +348,26 @@ public class NiFiTestUser {
* @throws java.lang.Exception ex
*/
public ClientResponse testDelete(String url) throws Exception {
- return testDelete(url, (Object) null);
+ return testDelete(url, null);
}
/**
* Performs a DELETE using the specified url and entity.
*
* @param url url
- * @param entity entity
- * @return repsonse
+ * @param headers http headers
+ * @return response
* @throws java.lang.Exception ex
*/
- public ClientResponse testDelete(String url, Object entity) throws Exception {
+ public ClientResponse testDeleteWithHeaders(String url, Map<String, String> headers) throws Exception {
// get the resource
- WebResource.Builder resourceBuilder = client.resource(url).accept(MediaType.APPLICATION_JSON).header(X509AuthenticationFilter.PROXY_ENTITIES_CHAIN, proxyDn);
+ WebResource.Builder resourceBuilder = addProxiedEntities(client.resource(url).accept(MediaType.APPLICATION_JSON));
- // append any query parameters
- if (entity != null) {
- resourceBuilder = resourceBuilder.entity(entity);
+ // append any headers
+ if (headers != null && !headers.isEmpty()) {
+ for (String key : headers.keySet()) {
+ resourceBuilder = resourceBuilder.header(key, headers.get(key));
+ }
}
// perform the query
@@ -255,7 +394,56 @@ public class NiFiTestUser {
}
// perform the request
- return resource.accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_FORM_URLENCODED).header(X509AuthenticationFilter.PROXY_ENTITIES_CHAIN, proxyDn).delete(ClientResponse.class);
+ return addProxiedEntities(resource.accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_FORM_URLENCODED)).delete(ClientResponse.class);
}
+ /**
+ * Attempts to create a token with the specified username and password.
+ *
+ * @param url the url
+ * @param username the username
+ * @param password the password
+ * @return response
+ * @throws Exception ex
+ */
+ public ClientResponse testCreateToken(String url, String username, String password) throws Exception {
+ // convert the form data
+ MultivaluedMapImpl entity = new MultivaluedMapImpl();
+ entity.add("username", username);
+ entity.add("password", password);
+
+ // get the resource
+ WebResource.Builder resourceBuilder = addProxiedEntities(client.resource(url).accept(MediaType.TEXT_PLAIN).type(MediaType.APPLICATION_FORM_URLENCODED)).entity(entity);
+
+ // perform the request
+ return resourceBuilder.post(ClientResponse.class);
+ }
+
+ /**
+ * Attempts to create a token with the specified username and password.
+ *
+ * @param url the url
+ * @param justification justification
+ * @param headers http headers
+ * @return response
+ * @throws Exception ex
+ */
+ public ClientResponse testRegisterUser(String url, String justification, Map<String, String> headers) throws Exception {
+ // convert the form data
+ MultivaluedMapImpl entity = new MultivaluedMapImpl();
+ entity.add("justification", justification);
+
+ // get the resource
+ WebResource.Builder resourceBuilder = addProxiedEntities(client.resource(url).accept(MediaType.TEXT_PLAIN).type(MediaType.APPLICATION_FORM_URLENCODED)).entity(entity);
+
+ // append any headers
+ if (headers != null && !headers.isEmpty()) {
+ for (String key : headers.keySet()) {
+ resourceBuilder = resourceBuilder.header(key, headers.get(key));
+ }
+ }
+
+ // perform the request
+ return resourceBuilder.post(ClientResponse.class);
+ }
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/META-INF/services/org.apache.nifi.authentication.LoginIdentityProvider
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/META-INF/services/org.apache.nifi.authentication.LoginIdentityProvider b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/META-INF/services/org.apache.nifi.authentication.LoginIdentityProvider
new file mode 100644
index 0000000..4b42e4f
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/META-INF/services/org.apache.nifi.authentication.LoginIdentityProvider
@@ -0,0 +1,15 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+org.apache.nifi.integration.util.NiFiTestLoginIdentityProvider
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/access-control/controller-services.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/access-control/controller-services.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/access-control/controller-services.xml
deleted file mode 100644
index f5bd96a..0000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/access-control/controller-services.xml
+++ /dev/null
@@ -1,18 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<services>
-
-</services>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/access-control/login-identity-providers.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/access-control/login-identity-providers.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/access-control/login-identity-providers.xml
new file mode 100644
index 0000000..04120c9
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/access-control/login-identity-providers.xml
@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!--
+ This file lists all authority providers to use when running securely.
+-->
+<loginIdentityProviders>
+ <provider>
+ <identifier>test-provider</identifier>
+ <class>org.apache.nifi.integration.util.NiFiTestLoginIdentityProvider</class>
+ </provider>
+</loginIdentityProviders>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/access-control/nifi.properties
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/access-control/nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/access-control/nifi.properties
index 0aa5a14..10db651 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/access-control/nifi.properties
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/access-control/nifi.properties
@@ -14,16 +14,15 @@
# limitations under the License.
# Core Properties #
-nifi.version=nifi 0.2.1-SNAPSHOT
+nifi.version=nifi version
nifi.flow.configuration.file=
nifi.flow.configuration.archive.dir=target/archive
nifi.flowcontroller.autoResumeState=true
nifi.flowcontroller.graceful.shutdown.period=10 sec
nifi.flowservice.writedelay.interval=2 sec
-nifi.reporting.task.configuration.file=target/test-classes/access-control/reporting-tasks.xml
-nifi.controller.service.configuration.file=target/test-classes/access-control/controller-services.xml
nifi.authority.provider.configuration.file=target/test-classes/access-control/authority-providers.xml
+nifi.login.identity.provider.configuration.file=target/test-classes/access-control/login-identity-providers.xml
nifi.templates.directory=target/test-classes/access-control/templates
nifi.ui.banner.text=TEST BANNER
nifi.ui.autorefresh.interval=30 sec
@@ -93,10 +92,11 @@ nifi.security.truststoreType=JKS
nifi.security.truststorePasswd=localtest
nifi.security.needClientAuth=true
nifi.security.user.authority.provider=test-provider
+nifi.security.user.login.identity.provider=test-provider
nifi.security.authorizedUsers.file=target/test-classes/access-control/users.xml
nifi.security.user.credential.cache.duration=1 hr
nifi.security.support.new.account.requests=
-nifi.security.default.user.roles=
+nifi.security.anonymous.authorities=
# cluster common properties (cluster manager and nodes must have same values) #
nifi.cluster.protocol.heartbeat.interval=5 sec
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/access-control/reporting-tasks.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/access-control/reporting-tasks.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/access-control/reporting-tasks.xml
deleted file mode 100644
index 251735e..0000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/access-control/reporting-tasks.xml
+++ /dev/null
@@ -1,17 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<tasks>
-</tasks>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/pom.xml
index daeef5c..ee18bff 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/pom.xml
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/pom.xml
@@ -22,6 +22,40 @@
</parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-web-security</artifactId>
+ <build>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ <resource>
+ <directory>src/main/xsd</directory>
+ </resource>
+ </resources>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>jaxb2-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>current</id>
+ <goals>
+ <goal>xjc</goal>
+ </goals>
+ <configuration>
+ <packageName>org.apache.nifi.authentication.generated</packageName>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <configuration>
+ <excludes>**/authentication/generated/*.java,</excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
@@ -29,6 +63,10 @@
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-nar-utils</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
<artifactId>nifi-api</artifactId>
</dependency>
<dependency>
@@ -40,6 +78,11 @@
<artifactId>nifi-framework-core</artifactId>
</dependency>
<dependency>
+ <groupId>io.jsonwebtoken</groupId>
+ <artifactId>jjwt</artifactId>
+ <version>0.6.0</version>
+ </dependency>
+ <dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk16</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/DnUtils.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/DnUtils.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/DnUtils.java
deleted file mode 100644
index f3bd11e..0000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/DnUtils.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.web.security;
-
-import java.security.cert.X509Certificate;
-import java.util.ArrayDeque;
-import java.util.Deque;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import javax.servlet.http.HttpServletRequest;
-import org.apache.nifi.web.security.x509.SubjectDnX509PrincipalExtractor;
-import org.apache.nifi.web.security.x509.X509CertificateExtractor;
-import org.apache.commons.lang3.StringUtils;
-
-/**
- *
- */
-public class DnUtils {
-
- private static final Pattern proxyChainPattern = Pattern.compile("<(.*?)>");
-
- /**
- * @param request http request
- * @return the X-ProxiedEntitiesChain from the specified request
- */
- public static String getXProxiedEntitiesChain(final HttpServletRequest request) {
- String xProxiedEntitiesChain = request.getHeader("X-ProxiedEntitiesChain");
- final X509Certificate cert = new X509CertificateExtractor().extractClientCertificate(request);
- if (cert != null) {
- final SubjectDnX509PrincipalExtractor principalExtractor = new SubjectDnX509PrincipalExtractor();
- final String extractedPrincipal = principalExtractor.extractPrincipal(cert).toString();
- final String formattedPrincipal = formatProxyDn(extractedPrincipal);
- if (StringUtils.isBlank(xProxiedEntitiesChain)) {
- xProxiedEntitiesChain = formattedPrincipal;
- } else {
- xProxiedEntitiesChain += formattedPrincipal;
- }
- }
-
- return xProxiedEntitiesChain;
- }
-
- /**
- * Formats the specified DN to be set as a HTTP header using well known
- * conventions.
- *
- * @param dn raw dn
- * @return the dn formatted as an HTTP header
- */
- public static String formatProxyDn(String dn) {
- return "<" + dn + ">";
- }
-
- /**
- * Tokenizes the specified proxy chain.
- *
- * @param rawProxyChain raw chain
- * @return tokenized proxy chain
- */
- public static Deque<String> tokenizeProxyChain(String rawProxyChain) {
- final Deque<String> dnList = new ArrayDeque<>();
-
- // parse the proxy chain
- final Matcher rawProxyChainMatcher = proxyChainPattern.matcher(rawProxyChain);
- while (rawProxyChainMatcher.find()) {
- dnList.push(rawProxyChainMatcher.group(1));
- }
-
- return dnList;
- }
-}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/InvalidAuthenticationException.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/InvalidAuthenticationException.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/InvalidAuthenticationException.java
new file mode 100644
index 0000000..1065152
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/InvalidAuthenticationException.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.security;
+
+import org.springframework.security.core.AuthenticationException;
+
+/**
+ * Thrown if the authentication of a given request is invalid. For instance,
+ * an expired certificate or token.
+ */
+public class InvalidAuthenticationException extends AuthenticationException {
+
+ public InvalidAuthenticationException(String msg) {
+ super(msg);
+ }
+
+ public InvalidAuthenticationException(String msg, Throwable t) {
+ super(msg, t);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/NiFiAuthenticationEntryPoint.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/NiFiAuthenticationEntryPoint.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/NiFiAuthenticationEntryPoint.java
new file mode 100644
index 0000000..ef1dfb2
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/NiFiAuthenticationEntryPoint.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.security;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import org.apache.nifi.util.NiFiProperties;
+import org.apache.nifi.util.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.security.core.AuthenticationException;
+import org.springframework.security.web.AuthenticationEntryPoint;
+
+/**
+ * This is our own implementation of org.springframework.security.web.AuthenticationEntryPoint that allows us to send the response to the client exactly how we want to and log the results.
+ */
+public class NiFiAuthenticationEntryPoint implements AuthenticationEntryPoint {
+
+ private static final Logger logger = LoggerFactory.getLogger(NiFiAuthenticationEntryPoint.class);
+
+ private final NiFiProperties properties;
+
+ public NiFiAuthenticationEntryPoint(NiFiProperties properties) {
+ this.properties = properties;
+ }
+
+ /**
+ * Always returns a 403 error code to the client.
+ *
+ * @param request request
+ * @param response response
+ * @param ae ae
+ * @throws java.io.IOException ex
+ * @throws javax.servlet.ServletException ex
+ */
+ @Override
+ public void commence(HttpServletRequest request, HttpServletResponse response, AuthenticationException ae) throws IOException, ServletException {
+ // if the content type is not set, mark as access denied
+ if (StringUtils.isBlank(response.getContentType())) {
+ // write the response message
+ PrintWriter out = response.getWriter();
+ response.setContentType("text/plain");
+
+ // return authorized if the request is secure and this nifi supports new account requests
+ if (request.isSecure() && properties.getSupportNewAccountRequests()) {
+ response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
+ out.println("Not authorized.");
+ } else {
+ response.setStatus(HttpServletResponse.SC_FORBIDDEN);
+ out.println("Access is denied.");
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/NiFiAuthenticationFilter.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/NiFiAuthenticationFilter.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/NiFiAuthenticationFilter.java
new file mode 100644
index 0000000..f09d610
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/NiFiAuthenticationFilter.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.security;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.user.NiFiUser;
+import org.apache.nifi.util.NiFiProperties;
+import org.apache.nifi.web.security.token.NiFiAuthenticationRequestToken;
+import org.apache.nifi.web.security.user.NiFiUserUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.security.authentication.AccountStatusException;
+import org.springframework.security.authentication.AuthenticationManager;
+import org.springframework.security.authentication.AuthenticationServiceException;
+import org.springframework.security.core.Authentication;
+import org.springframework.security.core.AuthenticationException;
+import org.springframework.security.core.context.SecurityContextHolder;
+import org.springframework.security.core.userdetails.UsernameNotFoundException;
+
+/**
+ *
+ */
+public abstract class NiFiAuthenticationFilter implements Filter {
+
+ private static final Logger logger = LoggerFactory.getLogger(NiFiAuthenticationFilter.class);
+
+ private AuthenticationManager authenticationManager;
+ private NiFiProperties properties;
+
+ @Override
+ public void init(final FilterConfig filterConfig) throws ServletException {
+ throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+ }
+
+ @Override
+ public void doFilter(final ServletRequest request, final ServletResponse response, final FilterChain chain) throws IOException, ServletException {
+ if (logger.isDebugEnabled()) {
+ logger.debug("Checking secure context token: " + SecurityContextHolder.getContext().getAuthentication());
+ }
+
+ if (requiresAuthentication((HttpServletRequest) request)) {
+ authenticate((HttpServletRequest) request, (HttpServletResponse) response);
+ }
+
+ chain.doFilter(request, response);
+ }
+
+ private boolean requiresAuthentication(final HttpServletRequest request) {
+ // continue attempting authorization if the user is anonymous
+ if (isAnonymousUser()) {
+ return true;
+ }
+
+ // or there is no user yet
+ return NiFiUserUtils.getNiFiUser() == null && NiFiUserUtils.getNewAccountRequest() == null;
+ }
+
+ private boolean isAnonymousUser() {
+ final NiFiUser user = NiFiUserUtils.getNiFiUser();
+ return user != null && NiFiUser.ANONYMOUS_USER_IDENTITY.equals(user.getIdentity());
+ }
+
+ private void authenticate(final HttpServletRequest request, final HttpServletResponse response) throws IOException {
+ try {
+ final NiFiAuthenticationRequestToken authenticated = attemptAuthentication(request, response);
+ if (authenticated != null) {
+ // log the request attempt - response details will be logged later
+ logger.info(String.format("Attempting request for (%s) %s %s (source ip: %s)",
+ ProxiedEntitiesUtils.formatProxyDn(StringUtils.join(authenticated.getChain(), "><")), request.getMethod(),
+ request.getRequestURL().toString(), request.getRemoteAddr()));
+
+ final Authentication authorized = authenticationManager.authenticate(authenticated);
+ successfulAuthorization(request, response, authorized);
+ }
+ } catch (final AuthenticationException ae) {
+ if (!isAnonymousUser()) {
+ unsuccessfulAuthorization(request, response, ae);
+ }
+ }
+ }
+
+ public abstract NiFiAuthenticationRequestToken attemptAuthentication(HttpServletRequest request, HttpServletResponse response);
+
+ protected void successfulAuthorization(HttpServletRequest request, HttpServletResponse response, Authentication authResult) {
+ if (logger.isDebugEnabled()) {
+ logger.debug("Authentication success: " + authResult);
+ }
+
+ SecurityContextHolder.getContext().setAuthentication(authResult);
+ ProxiedEntitiesUtils.successfulAuthorization(request, response, authResult);
+ }
+
+ protected void unsuccessfulAuthorization(HttpServletRequest request, HttpServletResponse response, AuthenticationException ae) throws IOException {
+ // populate the response
+ ProxiedEntitiesUtils.unsuccessfulAuthorization(request, response, ae);
+
+ // set the response status
+ response.setContentType("text/plain");
+
+ // write the response message
+ PrintWriter out = response.getWriter();
+
+ // use the type of authentication exception to determine the response code
+ if (ae instanceof UsernameNotFoundException) {
+ if (properties.getSupportNewAccountRequests()) {
+ response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
+ out.println("Not authorized.");
+ } else {
+ response.setStatus(HttpServletResponse.SC_FORBIDDEN);
+ out.println("Access is denied.");
+ }
+ } else if (ae instanceof InvalidAuthenticationException) {
+ response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
+ out.println(ae.getMessage());
+ } else if (ae instanceof AccountStatusException) {
+ response.setStatus(HttpServletResponse.SC_FORBIDDEN);
+ out.println(ae.getMessage());
+ } else if (ae instanceof UntrustedProxyException) {
+ response.setStatus(HttpServletResponse.SC_FORBIDDEN);
+ out.println(ae.getMessage());
+ } else if (ae instanceof AuthenticationServiceException) {
+ logger.error(String.format("Unable to authorize: %s", ae.getMessage()), ae);
+ response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+ out.println(String.format("Unable to authorize: %s", ae.getMessage()));
+ } else {
+ logger.error(String.format("Unable to authorize: %s", ae.getMessage()), ae);
+ response.setStatus(HttpServletResponse.SC_FORBIDDEN);
+ out.println("Access is denied.");
+ }
+
+ // log the failure
+ logger.info(String.format("Rejecting access to web api: %s", ae.getMessage()));
+
+ // optionally log the stack trace
+ if (logger.isDebugEnabled()) {
+ logger.debug(StringUtils.EMPTY, ae);
+ }
+ }
+
+ /**
+ * Determines if the specified request is attempting to register a new user account.
+ *
+ * @param request http request
+ * @return true if new user
+ */
+ protected final boolean isNewAccountRequest(HttpServletRequest request) {
+ if ("POST".equalsIgnoreCase(request.getMethod())) {
+ String path = request.getPathInfo();
+ if (StringUtils.isNotBlank(path)) {
+ if ("/controller/users".equals(path)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Extracts the justification from the specified request.
+ *
+ * @param request The request
+ * @return The justification
+ */
+ protected final String getJustification(HttpServletRequest request) {
+ // get the justification
+ String justification = request.getParameter("justification");
+ if (justification == null) {
+ justification = StringUtils.EMPTY;
+ }
+ return justification;
+ }
+
+ @Override
+ public void destroy() {
+ }
+
+ public void setAuthenticationManager(AuthenticationManager authenticationManager) {
+ this.authenticationManager = authenticationManager;
+ }
+
+ public void setProperties(NiFiProperties properties) {
+ this.properties = properties;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/NiFiAuthenticationProvider.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/NiFiAuthenticationProvider.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/NiFiAuthenticationProvider.java
new file mode 100644
index 0000000..eb0684b
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/NiFiAuthenticationProvider.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.security;
+
+import org.apache.nifi.web.security.token.NewAccountAuthenticationRequestToken;
+import org.apache.nifi.web.security.token.NewAccountAuthenticationToken;
+import org.apache.nifi.web.security.token.NiFiAuthenticationRequestToken;
+import org.apache.nifi.web.security.token.NiFiAuthorizationToken;
+import org.springframework.security.authentication.AuthenticationProvider;
+import org.springframework.security.core.Authentication;
+import org.springframework.security.core.AuthenticationException;
+import org.springframework.security.core.userdetails.AuthenticationUserDetailsService;
+import org.springframework.security.core.userdetails.UserDetails;
+import org.springframework.security.core.userdetails.UsernameNotFoundException;
+
+/**
+ *
+ */
+public class NiFiAuthenticationProvider implements AuthenticationProvider {
+
+ private final AuthenticationUserDetailsService<NiFiAuthenticationRequestToken> userDetailsService;
+
+ public NiFiAuthenticationProvider(final AuthenticationUserDetailsService<NiFiAuthenticationRequestToken> userDetailsService) {
+ this.userDetailsService = userDetailsService;
+ }
+
+ @Override
+ public Authentication authenticate(Authentication authentication) throws AuthenticationException {
+ final NiFiAuthenticationRequestToken request = (NiFiAuthenticationRequestToken) authentication;
+
+ try {
+ // defer to the nifi user details service to authorize the user
+ final UserDetails userDetails = userDetailsService.loadUserDetails(request);
+
+ // build an authentication for accesing nifi
+ final NiFiAuthorizationToken result = new NiFiAuthorizationToken(userDetails);
+ result.setDetails(request.getDetails());
+ return result;
+ } catch (final UsernameNotFoundException unfe) {
+ // if the authentication request is for a new account and it could not be authorized because the user was not found,
+ // return the token so the new account could be created. this must go here toe nsure that any proxies have been authorized
+ if (isNewAccountAuthenticationToken(request)) {
+ return new NewAccountAuthenticationToken(((NewAccountAuthenticationRequestToken) authentication).getNewAccountRequest());
+ } else {
+ throw unfe;
+ }
+ }
+ }
+
+ private boolean isNewAccountAuthenticationToken(final Authentication authentication) {
+ return NewAccountAuthenticationRequestToken.class.isAssignableFrom(authentication.getClass());
+ }
+
+ @Override
+ public boolean supports(Class<?> authentication) {
+ return NiFiAuthenticationRequestToken.class.isAssignableFrom(authentication);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/ProxiedEntitiesUtils.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/ProxiedEntitiesUtils.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/ProxiedEntitiesUtils.java
new file mode 100644
index 0000000..1b2f28a
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/ProxiedEntitiesUtils.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.security;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.user.NiFiUser;
+import org.springframework.security.core.Authentication;
+import org.springframework.security.core.AuthenticationException;
+
+/**
+ *
+ */
+public class ProxiedEntitiesUtils {
+
+ public static final String PROXY_ENTITIES_CHAIN = "X-ProxiedEntitiesChain";
+ public static final String PROXY_ENTITIES_ACCEPTED = "X-ProxiedEntitiesAccepted";
+ public static final String PROXY_ENTITIES_DETAILS = "X-ProxiedEntitiesDetails";
+
+ private static final Pattern proxyChainPattern = Pattern.compile("<(.*?)>");
+
+ /**
+ * Formats the specified DN to be set as a HTTP header using well known conventions.
+ *
+ * @param dn raw dn
+ * @return the dn formatted as an HTTP header
+ */
+ public static String formatProxyDn(String dn) {
+ return "<" + dn + ">";
+ }
+
+ /**
+ * Tokenizes the specified proxy chain.
+ *
+ * @param rawProxyChain raw chain
+ * @return tokenized proxy chain
+ */
+ public static List<String> tokenizeProxiedEntitiesChain(String rawProxyChain) {
+ final List<String> proxyChain = new ArrayList<>();
+ final Matcher rawProxyChainMatcher = proxyChainPattern.matcher(rawProxyChain);
+ while (rawProxyChainMatcher.find()) {
+ proxyChain.add(rawProxyChainMatcher.group(1));
+ }
+
+ return proxyChain;
+ }
+
+ /**
+ * Builds the proxy chain for the specified user.
+ *
+ * @param user The current user
+ * @return The proxy chain for that user in String form
+ */
+ public static String buildProxiedEntitiesChainString(final NiFiUser user) {
+ // calculate the dn chain
+ final List<String> proxyChain = buildProxiedEntitiesChain(user);
+ return formatProxyDn(StringUtils.join(proxyChain, "><"));
+ }
+
+ /**
+ * Builds the proxy chain for the specified user.
+ *
+ * @param user The current user
+ * @return The proxy chain for that user in List form
+ */
+ public static List<String> buildProxiedEntitiesChain(final NiFiUser user) {
+ // calculate the dn chain
+ final List<String> proxyChain = new ArrayList<>();
+
+ // build the dn chain
+ NiFiUser chainedUser = user;
+ do {
+ // add the entry for this user
+ proxyChain.add(chainedUser.getIdentity());
+
+ // go to the next user in the chain
+ chainedUser = chainedUser.getChain();
+ } while (chainedUser != null);
+
+ return proxyChain;
+ }
+
+ /**
+ * Builds the proxy chain from the specified request and user.
+ *
+ * @param request the request
+ * @param username the username
+ * @return the proxy chain in list form
+ */
+ public static List<String> buildProxiedEntitiesChain(final HttpServletRequest request, final String username) {
+ final String chain = buildProxiedEntitiesChainString(request, username);
+ return tokenizeProxiedEntitiesChain(chain);
+ }
+
+ /**
+ * Builds the dn chain from the specified request and user.
+ *
+ * @param request the request
+ * @param username the username
+ * @return the dn chain in string form
+ */
+ public static String buildProxiedEntitiesChainString(final HttpServletRequest request, final String username) {
+ String principal;
+ if (username.startsWith("<") && username.endsWith(">")) {
+ principal = username;
+ } else {
+ principal = formatProxyDn(username);
+ }
+
+ // look for a proxied user
+ if (StringUtils.isNotBlank(request.getHeader(PROXY_ENTITIES_CHAIN))) {
+ principal = request.getHeader(PROXY_ENTITIES_CHAIN) + principal;
+ }
+ return principal;
+ }
+
+ public static void successfulAuthorization(HttpServletRequest request, HttpServletResponse response, Authentication authResult) {
+ if (StringUtils.isNotBlank(request.getHeader(PROXY_ENTITIES_CHAIN))) {
+ response.setHeader(PROXY_ENTITIES_ACCEPTED, Boolean.TRUE.toString());
+ }
+ }
+
+ public static void unsuccessfulAuthorization(HttpServletRequest request, HttpServletResponse response, AuthenticationException failed) {
+ if (StringUtils.isNotBlank(request.getHeader(PROXY_ENTITIES_CHAIN))) {
+ response.setHeader(PROXY_ENTITIES_DETAILS, failed.getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/anonymous/NiFiAnonymousUserFilter.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/anonymous/NiFiAnonymousUserFilter.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/anonymous/NiFiAnonymousUserFilter.java
index 295f09c..e67ed2c 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/anonymous/NiFiAnonymousUserFilter.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/anonymous/NiFiAnonymousUserFilter.java
@@ -16,25 +16,22 @@
*/
package org.apache.nifi.web.security.anonymous;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.EnumSet;
import javax.servlet.http.HttpServletRequest;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.admin.service.AdministrationException;
import org.apache.nifi.admin.service.UserService;
+import org.apache.nifi.authorization.Authority;
import org.apache.nifi.user.NiFiUser;
import org.apache.nifi.web.security.user.NiFiUserDetails;
-import org.apache.nifi.util.NiFiProperties;
+import org.apache.nifi.web.security.token.NiFiAuthorizationToken;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.springframework.security.authentication.AnonymousAuthenticationToken;
import org.springframework.security.core.Authentication;
-import org.springframework.security.core.GrantedAuthority;
import org.springframework.security.web.authentication.AnonymousAuthenticationFilter;
/**
- * Custom AnonymouseAuthenticationFilter used to grant additional authorities
- * depending on the current operating mode.
+ * Custom AnonymouseAuthenticationFilter used to grant additional authorities depending on the current operating mode.
*/
public class NiFiAnonymousUserFilter extends AnonymousAuthenticationFilter {
@@ -42,7 +39,6 @@ public class NiFiAnonymousUserFilter extends AnonymousAuthenticationFilter {
private static final String ANONYMOUS_KEY = "anonymousNifiKey";
- private NiFiProperties properties;
private UserService userService;
public NiFiAnonymousUserFilter() {
@@ -51,51 +47,37 @@ public class NiFiAnonymousUserFilter extends AnonymousAuthenticationFilter {
@Override
protected Authentication createAuthentication(HttpServletRequest request) {
- Authentication authentication;
+ Authentication authentication = null;
+
try {
// load the anonymous user from the database
- NiFiUser user = userService.getUserByDn(NiFiUser.ANONYMOUS_USER_DN);
- NiFiUserDetails userDetails = new NiFiUserDetails(user);
+ NiFiUser user = userService.getUserByDn(NiFiUser.ANONYMOUS_USER_IDENTITY);
+
+ // if this is an unsecure request allow full access
+ if (!request.isSecure()) {
+ user.getAuthorities().addAll(EnumSet.allOf(Authority.class));
+ }
+
+ // only create an authentication token if the anonymous user has some authorities
+ if (!user.getAuthorities().isEmpty()) {
+ NiFiUserDetails userDetails = new NiFiUserDetails(user);
- // get the granted authorities
- List<GrantedAuthority> authorities = new ArrayList<>(userDetails.getAuthorities());
- authentication = new AnonymousAuthenticationToken(ANONYMOUS_KEY, userDetails, authorities);
+ // get the granted authorities
+ authentication = new NiFiAuthorizationToken(userDetails);
+ }
} catch (AdministrationException ase) {
// record the issue
anonymousUserFilterLogger.warn("Unable to load anonymous user from accounts database: " + ase.getMessage());
if (anonymousUserFilterLogger.isDebugEnabled()) {
anonymousUserFilterLogger.warn(StringUtils.EMPTY, ase);
}
-
- // defer to the base implementation
- authentication = super.createAuthentication(request);
}
return authentication;
}
- /**
- * Only supports anonymous users for non-secure requests or one way ssl.
- *
- * @param request request
- * @return true if allowed
- */
- @Override
- protected boolean applyAnonymousForThisRequest(HttpServletRequest request) {
- // anonymous for non secure requests
- if ("http".equalsIgnoreCase(request.getScheme())) {
- return true;
- }
-
- return !properties.getNeedClientAuth();
- }
-
/* setters */
public void setUserService(UserService userService) {
this.userService = userService;
}
- public void setProperties(NiFiProperties properties) {
- this.properties = properties;
- }
-
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/authentication/NiFiAuthenticationEntryPoint.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/authentication/NiFiAuthenticationEntryPoint.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/authentication/NiFiAuthenticationEntryPoint.java
deleted file mode 100644
index cd5f1ac..0000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/authentication/NiFiAuthenticationEntryPoint.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.web.security.authentication;
-
-import java.io.IOException;
-import java.io.PrintWriter;
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.security.core.AuthenticationException;
-import org.springframework.security.web.AuthenticationEntryPoint;
-import org.springframework.security.web.WebAttributes;
-
-/**
- * This is our own implementation of
- * org.springframework.security.web.AuthenticationEntryPoint that allows us to
- * send the response to the client exactly how we want to and log the results.
- */
-public class NiFiAuthenticationEntryPoint implements AuthenticationEntryPoint {
-
- private static final Logger logger = LoggerFactory.getLogger(NiFiAuthenticationEntryPoint.class);
-
- /**
- * Always returns a 403 error code to the client.
- *
- * @param request request
- * @param response response
- * @param ae ae
- * @throws java.io.IOException ex
- * @throws javax.servlet.ServletException ex
- */
- @Override
- public void commence(HttpServletRequest request, HttpServletResponse response, AuthenticationException ae) throws IOException, ServletException {
- // get the last exception - the exception that is being passed in is a generic no credentials found
- // exception because the authentication could not be found in the security context. the actual cause
- // of the problem is stored in the session as the authentication_exception
- Object authenticationException = request.getSession().getAttribute(WebAttributes.AUTHENTICATION_EXCEPTION);
-
- // log request result
- if (authenticationException instanceof AuthenticationException) {
- ae = (AuthenticationException) authenticationException;
- logger.info(String.format("Rejecting access to web api: %s", ae.getMessage()));
- }
-
- // set the response status
- response.setStatus(HttpServletResponse.SC_FORBIDDEN);
- response.setContentType("text/plain");
-
- // write the response message
- PrintWriter out = response.getWriter();
- out.println("Access is denied.");
- }
-}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/authorization/NiFiAuthorizationService.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/authorization/NiFiAuthorizationService.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/authorization/NiFiAuthorizationService.java
index 95b4669..23d9e61 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/authorization/NiFiAuthorizationService.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/authorization/NiFiAuthorizationService.java
@@ -16,34 +16,35 @@
*/
package org.apache.nifi.web.security.authorization;
-import java.util.Deque;
-import java.util.Iterator;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.ListIterator;
+import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.admin.service.AccountDisabledException;
import org.apache.nifi.admin.service.AccountNotFoundException;
import org.apache.nifi.admin.service.AccountPendingException;
import org.apache.nifi.admin.service.AdministrationException;
import org.apache.nifi.admin.service.UserService;
import org.apache.nifi.authorization.Authority;
-import org.apache.nifi.web.security.DnUtils;
import org.apache.nifi.user.NiFiUser;
-import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.util.NiFiProperties;
import org.apache.nifi.web.security.UntrustedProxyException;
import org.apache.nifi.web.security.user.NiFiUserDetails;
-import org.apache.nifi.util.NiFiProperties;
+import org.apache.nifi.web.security.token.NiFiAuthenticationRequestToken;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.dao.DataAccessException;
import org.springframework.security.authentication.AccountStatusException;
import org.springframework.security.authentication.AuthenticationServiceException;
import org.springframework.security.core.AuthenticationException;
+import org.springframework.security.core.userdetails.AuthenticationUserDetailsService;
import org.springframework.security.core.userdetails.UserDetails;
-import org.springframework.security.core.userdetails.UserDetailsService;
import org.springframework.security.core.userdetails.UsernameNotFoundException;
/**
* UserDetailsService that will verify user identity and grant user authorities.
*/
-public class NiFiAuthorizationService implements UserDetailsService {
+public class NiFiAuthorizationService implements AuthenticationUserDetailsService<NiFiAuthenticationRequestToken> {
private static final Logger logger = LoggerFactory.getLogger(NiFiAuthorizationService.class);
@@ -53,35 +54,33 @@ public class NiFiAuthorizationService implements UserDetailsService {
/**
* Loads the user details for the specified dn.
*
- * Synchronizing because we want each request to be authorized atomically
- * since each may contain any number of DNs. We wanted an access decision
- * made for each individual request as a whole (without other request
- * potentially impacting it).
+ * Synchronizing because we want each request to be authorized atomically since each may contain any number of DNs. We wanted an access decision made for each individual request as a whole
+ * (without other request potentially impacting it).
*
- * @param rawProxyChain proxy chain
+ * @param request request
* @return user details
* @throws UsernameNotFoundException ex
* @throws org.springframework.dao.DataAccessException ex
*/
@Override
- public synchronized UserDetails loadUserByUsername(String rawProxyChain) throws UsernameNotFoundException, DataAccessException {
+ public synchronized UserDetails loadUserDetails(NiFiAuthenticationRequestToken request) throws UsernameNotFoundException, DataAccessException {
NiFiUserDetails userDetails = null;
- final Deque<String> dnList = DnUtils.tokenizeProxyChain(rawProxyChain);
+ final List<String> chain = new ArrayList<>(request.getChain());
// ensure valid input
- if (dnList.size() == 0) {
- logger.warn("Malformed proxy chain: " + rawProxyChain);
+ if (chain.isEmpty()) {
+ logger.warn("Malformed proxy chain: " + StringUtils.join(request.getChain()));
throw new UntrustedProxyException("Malformed proxy chain.");
}
NiFiUser proxy = null;
// process each part of the proxy chain
- for (final Iterator<String> dnIter = dnList.iterator(); dnIter.hasNext();) {
- final String dn = dnIter.next();
+ for (final ListIterator<String> chainIter = request.getChain().listIterator(chain.size()); chainIter.hasPrevious();) {
+ final String dn = chainIter.previous();
// if there is another dn after this one, this dn is a proxy for the request
- if (dnIter.hasNext()) {
+ if (chainIter.hasPrevious()) {
try {
// get the user details for the proxy
final NiFiUserDetails proxyDetails = getNiFiUserDetails(dn);
@@ -108,9 +107,6 @@ public class NiFiAuthorizationService implements UserDetailsService {
// attempt to create a new user account for the proxying client
userService.createPendingUserAccount(dn, "Automatic account request generated for unknown proxy.");
-
- // propagate the exception to return the appropriate response
- throw new UsernameNotFoundException(String.format("An account request was generated for the proxy '%s'.", dn));
} catch (AdministrationException ae) {
throw new AuthenticationServiceException(String.format("Unable to create an account request for '%s': %s", dn, ae.getMessage()), ae);
} catch (IllegalArgumentException iae) {
@@ -121,10 +117,10 @@ public class NiFiAuthorizationService implements UserDetailsService {
throw new AccountStatusException(message) {
};
}
- } else {
- logger.warn(String.format("Untrusted proxy '%s' must be authorized with '%s' authority: %s", dn, Authority.ROLE_PROXY.toString(), unfe.getMessage()));
- throw new UntrustedProxyException(String.format("Untrusted proxy '%s' must be authorized with '%s'.", dn, Authority.ROLE_PROXY.toString()));
}
+
+ logger.warn(String.format("Untrusted proxy '%s' must be authorized with '%s' authority: %s", dn, Authority.ROLE_PROXY.toString(), unfe.getMessage()));
+ throw new UntrustedProxyException(String.format("Untrusted proxy '%s' must be authorized with '%s'.", dn, Authority.ROLE_PROXY.toString()));
} catch (AuthenticationException ae) {
logger.warn(String.format("Untrusted proxy '%s' must be authorized with '%s' authority: %s", dn, Authority.ROLE_PROXY.toString(), ae.getMessage()));
throw new UntrustedProxyException(String.format("Untrusted proxy '%s' must be authorized with '%s'.", dn, Authority.ROLE_PROXY.toString()));
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/authorization/NodeAuthorizedUserFilter.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/authorization/NodeAuthorizedUserFilter.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/authorization/NodeAuthorizedUserFilter.java
deleted file mode 100644
index 80feed7..0000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/authorization/NodeAuthorizedUserFilter.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.web.security.authorization;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.security.cert.X509Certificate;
-import javax.servlet.FilterChain;
-import javax.servlet.ServletException;
-import javax.servlet.ServletRequest;
-import javax.servlet.ServletResponse;
-import javax.servlet.http.HttpServletRequest;
-import org.apache.nifi.controller.FlowController;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.nifi.web.security.user.NiFiUserDetails;
-import org.apache.nifi.web.security.x509.SubjectDnX509PrincipalExtractor;
-import org.apache.nifi.web.security.x509.X509CertificateExtractor;
-import org.apache.nifi.user.NiFiUser;
-import org.apache.nifi.util.NiFiProperties;
-import org.apache.nifi.web.util.WebUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.context.ApplicationContext;
-import org.springframework.security.authentication.AuthenticationDetailsSource;
-import org.springframework.security.core.context.SecurityContextHolder;
-import org.springframework.security.web.authentication.WebAuthenticationDetailsSource;
-import org.springframework.security.web.authentication.preauth.PreAuthenticatedAuthenticationToken;
-import org.springframework.security.web.authentication.preauth.x509.X509PrincipalExtractor;
-import org.springframework.web.context.support.WebApplicationContextUtils;
-import org.springframework.web.filter.GenericFilterBean;
-
-/**
- * Custom filter to extract a user's authorities from the request where the user
- * was authenticated by the cluster manager and populate the threadlocal with
- * the authorized user. If the request contains the appropriate header with
- * authorities and the application instance is a node connected to the cluster,
- * then the authentication/authorization steps remaining in the filter chain are
- * skipped.
- *
- * Checking if the application instance is a connected node is important because
- * it prevents external clients from faking the request headers and bypassing
- * the authentication processing chain.
- */
-public class NodeAuthorizedUserFilter extends GenericFilterBean {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(NodeAuthorizedUserFilter.class);
-
- public static final String PROXY_USER_DETAILS = "X-ProxiedEntityUserDetails";
-
- private NiFiProperties properties;
- private final AuthenticationDetailsSource authenticationDetailsSource = new WebAuthenticationDetailsSource();
- private final X509CertificateExtractor certificateExtractor = new X509CertificateExtractor();
- private final X509PrincipalExtractor principalExtractor = new SubjectDnX509PrincipalExtractor();
-
- @Override
- public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException {
- final HttpServletRequest httpServletRequest = (HttpServletRequest) request;
-
- // get the proxied user's authorities
- final String hexEncodedUserDetails = httpServletRequest.getHeader(PROXY_USER_DETAILS);
-
- // check if the request has the necessary header information and this instance is configured as a node
- if (StringUtils.isNotBlank(hexEncodedUserDetails) && properties.isNode()) {
-
- // get the flow controller from the Spring context
- final ApplicationContext ctx = WebApplicationContextUtils.getWebApplicationContext(getServletContext());
- final FlowController flowController = ctx.getBean("flowController", FlowController.class);
-
- // check that we are connected to the cluster
- if (flowController.getNodeId() != null) {
- try {
- // get the DN from the cert in the request
- final X509Certificate certificate = certificateExtractor.extractClientCertificate((HttpServletRequest) request);
- if (certificate != null) {
- // extract the principal from the certificate
- final Object certificatePrincipal = principalExtractor.extractPrincipal(certificate);
- final String dn = certificatePrincipal.toString();
-
- // only consider the pre-authorized user when the request came from the NCM according to the DN in the certificate
- final String clusterManagerDN = flowController.getClusterManagerDN();
- if (clusterManagerDN != null && clusterManagerDN.equals(dn)) {
- // deserialize hex encoded object
- final Serializable userDetailsObj = WebUtils.deserializeHexToObject(hexEncodedUserDetails);
-
- // if we have a valid object, set the authentication token and bypass the remaining authentication processing chain
- if (userDetailsObj instanceof NiFiUserDetails) {
- final NiFiUserDetails userDetails = (NiFiUserDetails) userDetailsObj;
- final NiFiUser user = userDetails.getNiFiUser();
-
- // log the request attempt - response details will be logged later
- logger.info(String.format("Attempting request for (%s) %s %s (source ip: %s)", user.getDn(), httpServletRequest.getMethod(),
- httpServletRequest.getRequestURL().toString(), request.getRemoteAddr()));
-
- // we do not create the authentication token with the X509 certificate because the certificate is from the sending system, not the proxied user
- final PreAuthenticatedAuthenticationToken token = new PreAuthenticatedAuthenticationToken(userDetails, null, userDetails.getAuthorities());
- token.setDetails(authenticationDetailsSource.buildDetails(request));
- SecurityContextHolder.getContext().setAuthentication(token);
- }
- }
- }
- } catch (final ClassNotFoundException cnfe) {
- LOGGER.warn("Classpath issue detected because failed to deserialize authorized user in request header due to: " + cnfe, cnfe);
- }
- }
- }
-
- chain.doFilter(request, response);
- }
-
- /* setters */
- public void setProperties(NiFiProperties properties) {
- this.properties = properties;
- }
-}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/jwt/JwtAuthenticationFilter.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/jwt/JwtAuthenticationFilter.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/jwt/JwtAuthenticationFilter.java
new file mode 100644
index 0000000..20675fb
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/jwt/JwtAuthenticationFilter.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.security.jwt;
+
+import io.jsonwebtoken.JwtException;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.web.security.NiFiAuthenticationFilter;
+import org.apache.nifi.web.security.token.NewAccountAuthenticationRequestToken;
+import org.apache.nifi.web.security.token.NiFiAuthenticationRequestToken;
+import org.apache.nifi.web.security.user.NewAccountRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.util.Arrays;
+import org.apache.nifi.web.security.InvalidAuthenticationException;
+
+/**
+ */
+public class JwtAuthenticationFilter extends NiFiAuthenticationFilter {
+
+ private static final Logger logger = LoggerFactory.getLogger(JwtAuthenticationFilter.class);
+
+ private static final String AUTHORIZATION = "Authorization";
+
+ private JwtService jwtService;
+
+ @Override
+ public NiFiAuthenticationRequestToken attemptAuthentication(HttpServletRequest request, HttpServletResponse response) {
+ // only suppport jwt login when running securely
+ if (!request.isSecure()) {
+ return null;
+ }
+
+ // TODO: Refactor request header extraction logic to shared utility as it is duplicated in AccessResource
+
+ // get the principal out of the user token
+ final String authorization = request.getHeader(AUTHORIZATION);
+
+ // if there is no authorization header, we don't know the user
+ if (authorization == null) {
+ return null;
+ } else {
+ // Extract the Base64 encoded token from the Authorization header
+ final String token = StringUtils.substringAfterLast(authorization, " ");
+
+ try {
+ final String jwtPrincipal = jwtService.getAuthenticationFromToken(token);
+
+ if (isNewAccountRequest(request)) {
+ return new NewAccountAuthenticationRequestToken(new NewAccountRequest(Arrays.asList(jwtPrincipal), getJustification(request)));
+ } else {
+ return new NiFiAuthenticationRequestToken(Arrays.asList(jwtPrincipal));
+ }
+ } catch (JwtException e) {
+ throw new InvalidAuthenticationException(e.getMessage(), e);
+ }
+ }
+ }
+
+ public void setJwtService(JwtService jwtService) {
+ this.jwtService = jwtService;
+ }
+
+}
[17/50] [abbrv] nifi git commit: NIFI-1173: Even if FlowFile Queue is
empty,
it needs to hold onto the Empty Queue request so that subsequent cancel/clear
requests can reference it
Posted by mc...@apache.org.
NIFI-1173: Even if FlowFile Queue is empty, it needs to hold onto the Empty Queue request so that subsequent cancel/clear requests can reference it
Signed-off-by: Aldrin Piri <al...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/d88b6cb6
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/d88b6cb6
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/d88b6cb6
Branch: refs/heads/NIFI-655
Commit: d88b6cb6bca01507103c565d2489dd83ebae5e17
Parents: 180ea1b
Author: Mark Payne <ma...@hotmail.com>
Authored: Mon Nov 16 16:26:14 2015 -0500
Committer: Aldrin Piri <al...@apache.org>
Committed: Wed Nov 18 10:56:38 2015 -0500
----------------------------------------------------------------------
.../main/java/org/apache/nifi/controller/StandardFlowFileQueue.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/d88b6cb6/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
index 9a439c9..c6fbf28 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowFileQueue.java
@@ -862,6 +862,7 @@ public final class StandardFlowFileQueue implements FlowFileQueue {
if (originalSize.getObjectCount() == 0) {
dropRequest.setDroppedSize(originalSize);
dropRequest.setState(DropFlowFileState.COMPLETE);
+ dropRequestMap.put(requestIdentifier, dropRequest);
return dropRequest;
}
[07/50] [abbrv] nifi git commit: NIFI-1160: - Addressing possible NPE
when removing a drop request. - Only updating a connection status once it's
been retrieved for the first time.
Posted by mc...@apache.org.
NIFI-1160:
- Addressing possible NPE when removing a drop request.
- Only updating a connection status once it's been retrieved for the first time.
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/64369f67
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/64369f67
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/64369f67
Branch: refs/heads/NIFI-655
Commit: 64369f67f92c21be3845bf8766ccdd8cecd88adf
Parents: da9f40b
Author: Matt Gilman <ma...@gmail.com>
Authored: Mon Nov 16 08:34:04 2015 -0500
Committer: Matt Gilman <ma...@gmail.com>
Committed: Mon Nov 16 08:34:04 2015 -0500
----------------------------------------------------------------------
.../org/apache/nifi/web/dao/impl/StandardConnectionDAO.java | 8 +++++++-
.../nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js | 2 +-
2 files changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/64369f67/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java
index 66730a3..df271a5 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java
@@ -499,7 +499,13 @@ public class StandardConnectionDAO extends ComponentDAO implements ConnectionDAO
public DropFlowFileStatus deleteFlowFileDropRequest(String groupId, String connectionId, String dropRequestId) {
final Connection connection = locateConnection(groupId, connectionId);
final FlowFileQueue queue = connection.getFlowFileQueue();
- return queue.cancelDropFlowFileRequest(dropRequestId);
+
+ final DropFlowFileStatus dropFlowFileStatus = queue.cancelDropFlowFileRequest(dropRequestId);
+ if (dropFlowFileStatus == null) {
+ throw new ResourceNotFoundException(String.format("Unable to find drop request with id '%s'.", dropRequestId));
+ }
+
+ return dropFlowFileStatus;
}
/* setters */
http://git-wip-us.apache.org/repos/asf/nifi/blob/64369f67/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js
index 1b80700..e1ed3fe 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js
@@ -1018,7 +1018,7 @@ nf.Actions = (function () {
$('#drop-request-status-message').text(dropRequest.state);
// update the current number of enqueued flowfiles
- if (nf.Common.isDefinedAndNotNull(dropRequest.currentCount)) {
+ if (nf.Common.isDefinedAndNotNull(connection.status) && nf.Common.isDefinedAndNotNull(dropRequest.currentCount)) {
connection.status.queued = dropRequest.current;
nf.Connection.refresh(connection.id);
}
[35/50] [abbrv] nifi git commit: NIFI-1196 Correcting treatment of
FETCH events as a continuation of a lineage trail and not the start of a new
one.
Posted by mc...@apache.org.
NIFI-1196 Correcting treatment of FETCH events as a continuation of a lineage trail and not the start of a new one.
Reviewed by Tony Kurc (tkurc@apache.org)
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/1580edb5
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/1580edb5
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/1580edb5
Branch: refs/heads/NIFI-655
Commit: 1580edb558e96dc49a40b50e8f5fd7e31e61371d
Parents: cdd2c4f
Author: Aldrin Piri <al...@apache.org>
Authored: Sat Nov 21 17:59:34 2015 -0500
Committer: Tony Kurc <tr...@gmail.com>
Committed: Sat Nov 21 20:21:36 2015 -0500
----------------------------------------------------------------------
.../java/org/apache/nifi/provenance/StandardLineageResult.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/1580edb5/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardLineageResult.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardLineageResult.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardLineageResult.java
index cf16fc0..02c089d 100644
--- a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardLineageResult.java
+++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardLineageResult.java
@@ -259,6 +259,7 @@ public class StandardLineageResult implements ComputeLineageResult {
case FORK:
case JOIN:
case REPLAY:
+ case FETCH:
case CLONE: {
// For events that create FlowFile nodes, we need to create the FlowFile Nodes and associated Edges, as appropriate
for (final String childUuid : record.getChildUuids()) {
@@ -288,7 +289,6 @@ public class StandardLineageResult implements ComputeLineageResult {
}
break;
case RECEIVE:
- case FETCH:
case CREATE: {
// for a receive event, we want to create a FlowFile Node that represents the FlowFile received
// and create an edge from the Receive Event to the FlowFile Node
[49/50] [abbrv] nifi git commit: NIFI-655: - Refactoring web security
to use Spring Security Java Configuration. - Introducing security in Web UI
in order to get JWT.
Posted by mc...@apache.org.
NIFI-655:
- Refactoring web security to use Spring Security Java Configuration.
- Introducing security in Web UI in order to get JWT.
NIFI-655:
- Setting up the resources (js/css) for the login page.
NIFI-655:
- Adding support for configuring anonymous roles.
- Addressing checkstyle violations.
NIFI-655:
- Moving to token api to web-api.
- Creating an LoginProvider API for user/pass based authentication.
- Creating a module for funneling access to the authorized useres.
NIFI-655:
- Moving away from usage of DN to identity throughout the application (from the user db to the authorization provider).
- Updating the authorized users schema to support login users.
- Creating an extension point for authentication of users based on username/password.
NIFI-655:
- Creating an endpoint for returning the identity of the current user.
- Updating the LoginAuthenticationFilter.
NIFI-655:
- Moving NiFi registration to the login page.
- Running the authentication filters in a different order to ensure we can disambiguate each case.
- Starting to layout each case... Forbidden, Login, Create User, Create NiFi Account.
NIFI-655:
- Addressing checkstyle issues.
NIFI-655:
- Making nf-storage available in the login page.
- Requiring use of local storage.
- Ignoring security for GET requests when obtaining the login configuration.
NIFI-655:
- Adding a new endpoint to obtain the status of a user registration.
- Updated the login page loading to ensure all possible states work.
NIFI-655:
- Ensuring we know the necessary state before we attempt to render the login page.
- Building the proxy chain in the JWT authentication filter.
- Only rendering the login when appropriate.
NIFI-655:
- Starting to style the login page.
- Added simple 'login' support by identifying username/password. Issuing JWT token coming...
- Added logout support
- Rendering the username when appropriate.
NIFI-655:
- Extracting certificate validation into a utility class.
- Fixing checkstyle issues.
- Cleaning up the web security context.
- Removing proxy chain checking where possible.
NIFI-655:
- Starting to add support for registration.
- Creating registration form.
NIFI-655:
- Starting to implement the JWT service.
- Parsing JWT on client side in order to render who the user currently is when logged in.
NIFI-655:
- Allowing the user to link back to the log in page from the new account page.
- Renaming DN to identity where possible.
NIFI-655:
- Fixing checkstyle issues.
NIFI-655:
- Adding more/better support for logging out.
NIFI-655:
- Fixing checkstyle issues.
NIFI-655:
- Adding a few new exceptions for the login identity provider.
NIFI-655:
- Disabling log in by default initially.
- Restoring authorization service unit test.
NIFI-655:
- Fixing checkstyle issues.
NIFI-655:
- Updating packages for log in filters.
- Handling new registration exceptions.
- Code clean up.
NIFI-655:
- Removing registration support.
- Removing file based implementation.
NIFI-655:
- Removing file based implementation.
NIFI-655:
- Removing unused spring configuration files.
NIFI-655:
- Making the auto wiring more explicit.
NIFI-655:
- Removing unused dependencies.
NIFI-655:
- Removing unused filter.
NIFI-655:
- Updating the login API authenticate method to use a richer set of exceptions.
- UI code clean.
NIFI-655:
- Ensuring the login identity provider is able to switch context classloaders via the standard NAR mechanisms.
NIFI-655:
- Initial commit of the LDAP based identity providers.
- Fixed issue when attempting to log into a NiFi that does not support new account requests.
NIFI-655:
- Allowing the ldap provider to specify if client authentication is required/desired.
NIFI-655:
- Persisting keys to sign user tokens.
- Allowing the identity provider to specify the token expiration.
- Code clean up.
NIFI-655:
- Ensuring identities are unique in the key table.
NIFI-655:
- Adding support for specifying the user search base and user search filter in the active directory provider.
NIFI-655:
- Fixing checkstyle issues.
NIFI-655:
- Adding automatic client side token renewal.
NIFI-655:
- Ensuring the logout link is rendered when appropriate.
NIFI-655:
- Adding configuration options for referrals and connect/read timeouts
NIFI-655:
- Added an endpoint for access details including configuration, creating tokens, and checking status.
- Updated DTOs and client side to utilize new endpoints.
NIFI-655:
- Refactoring certificate extraction and validation.
- Refactoring how expiration is specified in the login identity providers.
- Adding unit tests for the access endpoints.
- Code clean up.
NIFI-655:
- Keeping token expiration between 1 minute and 12 hours.
NIFI-655:
- Using the user identity provided by the login identity provider.
NIFI-655: - Fixed typo in error message for unrecognized authentication strategy.
Signed-off-by: Matt Gilman <ma...@gmail.com>
NIFI-655. - Added logback-test.xml configuration resource for nifi-web-security.
Signed-off-by: Matt Gilman <ma...@gmail.com>
NIFI-655. - Added issuer field to LoginAuthenticationToken. - Updated AccessResource to pass identity provider class name when creating LoginAuthenticationTokens. - Began refactoring JWT logic from request parsing logic in JwtService. - Added unit tests for JWT logic.
Signed-off-by: Matt Gilman <ma...@gmail.com>
NIFI-655. - Changed issuer field to use FQ class name because some classes return an empty string for getSimpleName(). - Finished refactoring JWT logic from request parsing logic in JwtService. - Updated AccessResource and JwtAuthenticationFilter to call new JwtService methods decoupled from request header parsing. - Added extensive unit tests for JWT logic.
Signed-off-by: Matt Gilman <ma...@gmail.com>
NIFI-655:
- Refactoring key service to expose the key id.
- Handling client side expiration better.
- Removing specialized active directory provider and abstract ldap provider.
NIFI-655. - Updated JwtService and JwtServiceTest to use Key POJO instead of raw String key from KeyService.
Signed-off-by: Matt Gilman <ma...@gmail.com>
NIFI-655:
- Fixing typo when loading the ldap connect timeout.
- Providing a better experience for session expiration.
- Using ellipsis for lengthly user name.
- Adding an issuer to the authentication response so the LIP can specify the appropriate value.
NIFI-655:
- Showing a logging in notification during the log in process.
NIFI-655:
- Removing unnecessary class.
NIFI-655:
- Fixing checkstyle issues.
- Showing the progress spinner while submitting account justification.
NIFI-655:
- Removing deprecated authentication strategy.
- Renaming TLS to START_TLS.
- Allowing the protocol to be configured.
NIFI-655:
- Fixing issue detecting the presence of DN column
NIFI-655:
- Pre-populating the login-identity-providers.xml file with necessary properties and documentation.
- Renaming the Authentication Duration property name.
NIFI-655:
- Updating documentation for the failure response codes.
NIFI-655:
- Ensuring the user identity is not too long.
NIFI-655:
- Updating default authentication expiration to 12 hours.
NIFI-655:
- Remaining on the login form when there is any unsuccessful login attempt.
- Fixing checkstyle issues.
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/aaf14c45
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/aaf14c45
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/aaf14c45
Branch: refs/heads/NIFI-655
Commit: aaf14c45c96077c0075af8f3442e392b717244c1
Parents: 4e2c94d
Author: Matt Gilman <ma...@gmail.com>
Authored: Wed Oct 7 13:33:34 2015 -0400
Committer: Matt Gilman <ma...@gmail.com>
Committed: Mon Nov 23 14:50:13 2015 -0500
----------------------------------------------------------------------
LICENSE | 22 +
.../authentication/AuthenticationResponse.java | 65 +
.../nifi/authentication/LoginCredentials.java | 39 +
.../authentication/LoginIdentityProvider.java | 61 +
...ginIdentityProviderConfigurationContext.java | 48 +
...inIdentityProviderInitializationContext.java | 27 +
.../LoginIdentityProviderLookup.java | 25 +
.../LoginIdentityProviderContext.java | 35 +
.../exception/IdentityAccessException.java | 33 +
.../InvalidLoginCredentialsException.java | 33 +
.../nifi/authorization/AuthorityProvider.java | 42 +-
.../nifi/web/NiFiWebConfigurationContext.java | 2 +-
.../org/apache/nifi/web/NiFiWebContext.java | 2 +-
nifi-assembly/LICENSE | 22 +
nifi-assembly/pom.xml | 8 +
.../org/apache/nifi/util/NiFiProperties.java | 64 +-
.../nifi/security/util/CertificateUtils.java | 51 +-
.../nifi-framework/nifi-administration/pom.xml | 1 -
.../nifi/admin/AuditDataSourceFactoryBean.java | 23 +-
.../nifi/admin/KeyDataSourceFactoryBean.java | 154 ++
.../nifi/admin/UserDataSourceFactoryBean.java | 93 +-
.../org/apache/nifi/admin/dao/DAOFactory.java | 2 +
.../java/org/apache/nifi/admin/dao/KeyDAO.java | 49 +
.../nifi/admin/dao/impl/DAOFactoryImpl.java | 6 +
.../nifi/admin/dao/impl/StandardActionDAO.java | 10 +-
.../nifi/admin/dao/impl/StandardKeyDAO.java | 154 ++
.../nifi/admin/dao/impl/StandardUserDAO.java | 53 +-
.../apache/nifi/admin/service/KeyService.java | 42 +
.../service/action/AbstractUserAction.java | 4 +-
.../service/action/AuthorizeUserAction.java | 24 +-
.../admin/service/action/DisableUserAction.java | 6 +-
.../admin/service/action/GetKeyByIdAction.java | 42 +
.../service/action/GetKeyByIdentityAction.java | 42 +
.../service/action/GetOrCreateKeyAction.java | 48 +
.../action/RequestUserAccountAction.java | 14 +-
.../service/action/SeedUserAccountsAction.java | 18 +-
.../admin/service/action/UngroupUserAction.java | 6 +-
.../admin/service/action/UpdateUserAction.java | 24 +-
.../service/action/UpdateUserGroupAction.java | 32 +-
.../admin/service/impl/StandardKeyService.java | 126 ++
.../src/main/java/org/apache/nifi/key/Key.java | 69 +
.../java/org/apache/nifi/user/NiFiUser.java | 18 +-
.../resources/nifi-administration-context.xml | 16 +
.../service/action/AuthorizeUserActionTest.java | 100 +-
.../service/action/CreateUserActionTest.java | 12 +-
.../service/action/DisableUserActionTest.java | 14 +-
.../action/RequestUserAccountActionTest.java | 22 +-
.../action/SeedUserAccountsActionTest.java | 44 +-
.../action/SetUserAuthoritiesActionTest.java | 18 +-
.../web/api/dto/AccessConfigurationDTO.java | 61 +
.../nifi/web/api/dto/AccessStatusDTO.java | 101 +
.../api/entity/AccessConfigurationEntity.java | 43 +
.../nifi/web/api/entity/AccessStatusEntity.java | 43 +
.../nifi/web/api/entity/IdentityEntity.java | 52 +
.../org/apache/nifi/nar/ExtensionManager.java | 17 +-
.../nifi/nar/NarThreadContextClassLoader.java | 2 +
.../resources/conf/login-identity-providers.xml | 92 +
.../src/main/resources/conf/nifi.properties | 3 +
.../org/apache/nifi/web/server/JettyServer.java | 8 +-
.../apache/nifi/audit/ControllerAuditor.java | 8 +-
.../nifi/audit/ControllerServiceAuditor.java | 12 +-
.../org/apache/nifi/audit/FunnelAuditor.java | 2 +-
.../java/org/apache/nifi/audit/PortAuditor.java | 6 +-
.../apache/nifi/audit/ProcessGroupAuditor.java | 6 +-
.../org/apache/nifi/audit/ProcessorAuditor.java | 6 +-
.../apache/nifi/audit/RelationshipAuditor.java | 4 +-
.../nifi/audit/RemoteProcessGroupAuditor.java | 6 +-
.../apache/nifi/audit/ReportingTaskAuditor.java | 6 +-
.../org/apache/nifi/audit/SnippetAuditor.java | 2 +-
.../org/apache/nifi/web/NiFiServiceFacade.java | 7 +
.../nifi/web/NiFiWebApiConfiguration.java | 40 +
.../web/NiFiWebApiSecurityConfiguration.java | 179 ++
.../nifi/web/StandardNiFiServiceFacade.java | 28 +-
.../StandardNiFiWebConfigurationContext.java | 8 +-
.../apache/nifi/web/StandardNiFiWebContext.java | 8 +-
.../org/apache/nifi/web/api/AccessResource.java | 424 ++++
.../nifi/web/api/ApplicationResource.java | 16 +-
.../apache/nifi/web/api/ControllerResource.java | 65 +-
.../org/apache/nifi/web/api/UserResource.java | 49 +-
.../api/config/AccessDeniedExceptionMapper.java | 2 +-
.../InvalidAuthenticationExceptionMapper.java | 44 +
.../org/apache/nifi/web/api/dto/DtoFactory.java | 2 +-
.../nifi/web/controller/ControllerFacade.java | 17 +-
.../web/dao/impl/StandardConnectionDAO.java | 2 +-
.../apache/nifi/web/filter/RequestLogger.java | 6 +-
.../src/main/resources/nifi-web-api-context.xml | 9 +
.../src/main/webapp/WEB-INF/web.xml | 13 +-
.../accesscontrol/AccessTokenEndpointTest.java | 292 +++
.../util/NiFiTestAuthorizationProvider.java | 3 +-
.../util/NiFiTestLoginIdentityProvider.java | 75 +
.../nifi/integration/util/NiFiTestServer.java | 9 +-
.../nifi/integration/util/NiFiTestUser.java | 232 ++-
...he.nifi.authentication.LoginIdentityProvider | 15 +
.../access-control/controller-services.xml | 18 -
.../access-control/login-identity-providers.xml | 24 +
.../resources/access-control/nifi.properties | 8 +-
.../access-control/reporting-tasks.xml | 17 -
.../nifi-web/nifi-web-security/pom.xml | 43 +
.../org/apache/nifi/web/security/DnUtils.java | 85 -
.../InvalidAuthenticationException.java | 35 +
.../security/NiFiAuthenticationEntryPoint.java | 71 +
.../web/security/NiFiAuthenticationFilter.java | 209 ++
.../security/NiFiAuthenticationProvider.java | 73 +
.../nifi/web/security/ProxiedEntitiesUtils.java | 147 ++
.../anonymous/NiFiAnonymousUserFilter.java | 56 +-
.../NiFiAuthenticationEntryPoint.java | 69 -
.../authorization/NiFiAuthorizationService.java | 46 +-
.../authorization/NodeAuthorizedUserFilter.java | 128 --
.../security/jwt/JwtAuthenticationFilter.java | 80 +
.../nifi/web/security/jwt/JwtService.java | 162 ++
.../security/node/NodeAuthorizedUserFilter.java | 127 ++
.../LoginIdentityProviderFactoryBean.java | 312 +++
...ginIdentityProviderConfigurationContext.java | 51 +
...inIdentityProviderInitializationContext.java | 45 +
.../token/LoginAuthenticationToken.java | 123 ++
.../NewAccountAuthenticationRequestToken.java | 40 +
.../token/NewAccountAuthenticationToken.java | 46 +
.../token/NiFiAuthenticationRequestToken.java | 54 +
.../security/token/NiFiAuthorizationToken.java | 50 +
.../web/security/user/NewAccountRequest.java | 47 +
.../nifi/web/security/user/NiFiUserDetails.java | 3 +-
.../nifi/web/security/user/NiFiUserUtils.java | 27 +-
.../security/x509/X509AuthenticationFilter.java | 304 +--
.../security/x509/X509CertificateExtractor.java | 4 +-
.../security/x509/X509CertificateValidator.java | 58 +
.../web/security/x509/X509IdentityProvider.java | 94 +
.../x509/ocsp/OcspCertificateValidator.java | 20 +-
.../resources/nifi-web-security-context.xml | 83 +-
.../src/main/xsd/login-identity-providers.xsd | 49 +
.../NiFiAuthorizationServiceTest.java | 64 +-
.../nifi/web/security/jwt/JwtServiceTest.java | 445 ++++
.../src/test/resources/logback-test.xml | 36 +
.../nifi-framework/nifi-web/nifi-web-ui/pom.xml | 53 +-
.../src/main/resources/META-INF/LICENSE | 22 +
.../resources/filters/bulletin-board.properties | 1 +
.../main/resources/filters/canvas.properties | 3 +-
.../main/resources/filters/cluster.properties | 1 +
.../main/resources/filters/counters.properties | 1 +
.../main/resources/filters/history.properties | 1 +
.../main/resources/filters/login-min.properties | 18 +
.../src/main/resources/filters/login.properties | 25 +
.../resources/filters/provenance.properties | 1 +
.../main/resources/filters/summary.properties | 1 +
.../main/resources/filters/templates.properties | 1 +
.../src/main/resources/filters/users.properties | 1 +
.../webapp/WEB-INF/pages/bulletin-board.jsp | 1 +
.../src/main/webapp/WEB-INF/pages/canvas.jsp | 3 +-
.../src/main/webapp/WEB-INF/pages/cluster.jsp | 1 +
.../src/main/webapp/WEB-INF/pages/counters.jsp | 1 +
.../src/main/webapp/WEB-INF/pages/history.jsp | 1 +
.../src/main/webapp/WEB-INF/pages/login.jsp | 54 +
.../main/webapp/WEB-INF/pages/message-page.jsp | 4 +-
.../main/webapp/WEB-INF/pages/provenance.jsp | 1 +
.../src/main/webapp/WEB-INF/pages/summary.jsp | 1 +
.../src/main/webapp/WEB-INF/pages/templates.jsp | 1 +
.../src/main/webapp/WEB-INF/pages/users.jsp | 1 +
.../WEB-INF/partials/canvas/canvas-header.jsp | 11 +
.../WEB-INF/partials/canvas/registration.jsp | 44 -
.../WEB-INF/partials/login/login-form.jsp | 32 +
.../WEB-INF/partials/login/login-message.jsp | 21 +
.../WEB-INF/partials/login/login-progress.jsp | 22 +
.../WEB-INF/partials/login/login-submission.jsp | 20 +
.../partials/login/nifi-registration-form.jsp | 38 +
.../webapp/WEB-INF/partials/message-pane.jsp | 7 +-
.../partials/users/user-details-dialog.jsp | 2 +-
.../nifi-web-ui/src/main/webapp/WEB-INF/web.xml | 22 +-
.../nifi-web-ui/src/main/webapp/css/canvas.css | 1 -
.../nifi-web-ui/src/main/webapp/css/header.css | 18 +
.../nifi-web-ui/src/main/webapp/css/login.css | 95 +
.../nifi-web-ui/src/main/webapp/css/main.css | 13 +-
.../src/main/webapp/css/registration.css | 45 -
.../src/main/webapp/js/jquery/jquery.base64.js | 123 ++
.../js/nf/bulletin-board/nf-bulletin-board.js | 2 +
.../webapp/js/nf/canvas/nf-canvas-header.js | 49 +-
.../src/main/webapp/js/nf/canvas/nf-canvas.js | 248 ++-
.../main/webapp/js/nf/canvas/nf-registration.js | 71 -
.../src/main/webapp/js/nf/canvas/nf-storage.js | 139 --
.../src/main/webapp/js/nf/cluster/nf-cluster.js | 2 +
.../main/webapp/js/nf/counters/nf-counters.js | 2 +
.../src/main/webapp/js/nf/history/nf-history.js | 2 +
.../src/main/webapp/js/nf/login/nf-login.js | 302 +++
.../src/main/webapp/js/nf/nf-common.js | 1926 ++++++++++--------
.../src/main/webapp/js/nf/nf-dialog.js | 23 +-
.../src/main/webapp/js/nf/nf-storage.js | 172 ++
.../webapp/js/nf/provenance/nf-provenance.js | 2 +
.../src/main/webapp/js/nf/summary/nf-summary.js | 2 +
.../main/webapp/js/nf/templates/nf-templates.js | 2 +
.../src/main/webapp/js/nf/users/nf-users.js | 2 +
.../nifi-ldap-iaa-providers-nar/pom.xml | 32 +
.../nifi-ldap-iaa-providers/pom.xml | 60 +
.../nifi/ldap/LdapAuthenticationStrategy.java | 27 +
.../java/org/apache/nifi/ldap/LdapProvider.java | 279 +++
.../org/apache/nifi/ldap/ReferralStrategy.java | 39 +
...he.nifi.authentication.LoginIdentityProvider | 15 +
.../nifi-ldap-iaa-providers-bundle/pom.xml | 38 +
nifi-nar-bundles/pom.xml | 1 +
pom.xml | 67 +-
197 files changed, 8863 insertions(+), 2596 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/LICENSE
----------------------------------------------------------------------
diff --git a/LICENSE b/LICENSE
index 59741e6..f4be753 100644
--- a/LICENSE
+++ b/LICENSE
@@ -374,6 +374,28 @@ For details see http://jqueryui.com
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+This product bundles 'jquery.base64.js' which is available under an MIT style license.
+
+ Copyright (c) 2013 Yannick Albert (http://yckart.com/)
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the "Software"),
+ to deal in the Software without restriction, including without limitation
+ the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ sell copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
This product bundles 'SlickGrid v2.2' which is available under an MIT style license.
Copyright (c) 2010 Michael Leibman, http://github.com/mleibman/slickgrid
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-api/src/main/java/org/apache/nifi/authentication/AuthenticationResponse.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/authentication/AuthenticationResponse.java b/nifi-api/src/main/java/org/apache/nifi/authentication/AuthenticationResponse.java
new file mode 100644
index 0000000..e9999fc
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/authentication/AuthenticationResponse.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.authentication;
+
+/**
+ * Authentication response for a user login attempt.
+ */
+public class AuthenticationResponse {
+
+ private final String identity;
+ private final String username;
+ private final long expiration;
+ private final String issuer;
+
+ /**
+ * Creates an authentication response. The username and how long the authentication is valid in milliseconds
+ *
+ * @param identity The user identity
+ * @param username The username
+ * @param expiration The expiration in milliseconds
+ * @param issuer The issuer of the token
+ */
+ public AuthenticationResponse(final String identity, final String username, final long expiration, final String issuer) {
+ this.identity = identity;
+ this.username = username;
+ this.expiration = expiration;
+ this.issuer = issuer;
+ }
+
+ public String getIdentity() {
+ return identity;
+ }
+
+ public String getUsername() {
+ return username;
+ }
+
+ public String getIssuer() {
+ return issuer;
+ }
+
+ /**
+ * Returns the expiration of a given authentication in milliseconds.
+ *
+ * @return The expiration in milliseconds
+ */
+ public long getExpiration() {
+ return expiration;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-api/src/main/java/org/apache/nifi/authentication/LoginCredentials.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/authentication/LoginCredentials.java b/nifi-api/src/main/java/org/apache/nifi/authentication/LoginCredentials.java
new file mode 100644
index 0000000..afb7827
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/authentication/LoginCredentials.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.authentication;
+
+/**
+ * Login credentials for a user.
+ */
+public class LoginCredentials {
+
+ private final String username;
+ private final String password;
+
+ public LoginCredentials(String username, String password) {
+ this.username = username;
+ this.password = password;
+ }
+
+ public String getUsername() {
+ return username;
+ }
+
+ public String getPassword() {
+ return password;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-api/src/main/java/org/apache/nifi/authentication/LoginIdentityProvider.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/authentication/LoginIdentityProvider.java b/nifi-api/src/main/java/org/apache/nifi/authentication/LoginIdentityProvider.java
new file mode 100644
index 0000000..54becb3
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/authentication/LoginIdentityProvider.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.authentication;
+
+import org.apache.nifi.authentication.exception.IdentityAccessException;
+import org.apache.nifi.authentication.exception.InvalidLoginCredentialsException;
+import org.apache.nifi.authorization.exception.ProviderCreationException;
+import org.apache.nifi.authorization.exception.ProviderDestructionException;
+
+/**
+ * Identity provider that is able to authentication a user with username/password credentials.
+ */
+public interface LoginIdentityProvider {
+
+ /**
+ * Authenticates the specified login credentials.
+ *
+ * @param credentials the credentials
+ * @return The authentication response
+ * @throws InvalidLoginCredentialsException The login credentials were invalid
+ * @throws IdentityAccessException Unable to register the user due to an issue accessing the underlying storage
+ */
+ AuthenticationResponse authenticate(LoginCredentials credentials) throws InvalidLoginCredentialsException, IdentityAccessException;
+
+ /**
+ * Called immediately after instance creation for implementers to perform additional setup
+ *
+ * @param initializationContext in which to initialize
+ * @throws ProviderCreationException Unable to initialize
+ */
+ void initialize(LoginIdentityProviderInitializationContext initializationContext) throws ProviderCreationException;
+
+ /**
+ * Called to configure the AuthorityProvider.
+ *
+ * @param configurationContext at the time of configuration
+ * @throws ProviderCreationException for any issues configuring the provider
+ */
+ void onConfigured(LoginIdentityProviderConfigurationContext configurationContext) throws ProviderCreationException;
+
+ /**
+ * Called immediately before instance destruction for implementers to release resources.
+ *
+ * @throws ProviderDestructionException If pre-destruction fails.
+ */
+ void preDestruction() throws ProviderDestructionException;
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-api/src/main/java/org/apache/nifi/authentication/LoginIdentityProviderConfigurationContext.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/authentication/LoginIdentityProviderConfigurationContext.java b/nifi-api/src/main/java/org/apache/nifi/authentication/LoginIdentityProviderConfigurationContext.java
new file mode 100644
index 0000000..6fe61fc
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/authentication/LoginIdentityProviderConfigurationContext.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.authentication;
+
+import java.util.Map;
+
+/**
+ *
+ */
+public interface LoginIdentityProviderConfigurationContext {
+
+ /**
+ * @return identifier for the authority provider
+ */
+ String getIdentifier();
+
+ /**
+ * Retrieves all properties the component currently understands regardless
+ * of whether a value has been set for them or not. If no value is present
+ * then its value is null and thus any registered default for the property
+ * descriptor applies.
+ *
+ * @return Map of all properties
+ */
+ Map<String, String> getProperties();
+
+ /**
+ * @param property to lookup the descriptor and value of
+ * @return the value the component currently understands for the given
+ * PropertyDescriptor. This method does not substitute default
+ * PropertyDescriptor values, so the value returned will be null if not set
+ */
+ String getProperty(String property);
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-api/src/main/java/org/apache/nifi/authentication/LoginIdentityProviderInitializationContext.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/authentication/LoginIdentityProviderInitializationContext.java b/nifi-api/src/main/java/org/apache/nifi/authentication/LoginIdentityProviderInitializationContext.java
new file mode 100644
index 0000000..7aa72a7
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/authentication/LoginIdentityProviderInitializationContext.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.authentication;
+
+/**
+ *
+ */
+public interface LoginIdentityProviderInitializationContext {
+
+ public String getIdentifier();
+
+ public LoginIdentityProviderLookup getAuthorityProviderLookup();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-api/src/main/java/org/apache/nifi/authentication/LoginIdentityProviderLookup.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/authentication/LoginIdentityProviderLookup.java b/nifi-api/src/main/java/org/apache/nifi/authentication/LoginIdentityProviderLookup.java
new file mode 100644
index 0000000..3cd0ba7
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/authentication/LoginIdentityProviderLookup.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.authentication;
+
+/**
+ *
+ */
+public interface LoginIdentityProviderLookup {
+
+ LoginIdentityProvider getLoginIdentityProvider(String identifier);
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-api/src/main/java/org/apache/nifi/authentication/annotation/LoginIdentityProviderContext.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/authentication/annotation/LoginIdentityProviderContext.java b/nifi-api/src/main/java/org/apache/nifi/authentication/annotation/LoginIdentityProviderContext.java
new file mode 100644
index 0000000..88df259
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/authentication/annotation/LoginIdentityProviderContext.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.authentication.annotation;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ *
+ *
+ */
+@Documented
+@Target({ElementType.FIELD, ElementType.METHOD})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+public @interface LoginIdentityProviderContext {
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-api/src/main/java/org/apache/nifi/authentication/exception/IdentityAccessException.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/authentication/exception/IdentityAccessException.java b/nifi-api/src/main/java/org/apache/nifi/authentication/exception/IdentityAccessException.java
new file mode 100644
index 0000000..b68c675
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/authentication/exception/IdentityAccessException.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.authentication.exception;
+
+/**
+ * Represents the case when the identity could not be confirmed because it was unable
+ * to access the backing store.
+ */
+public class IdentityAccessException extends RuntimeException {
+
+ public IdentityAccessException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public IdentityAccessException(String message) {
+ super(message);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-api/src/main/java/org/apache/nifi/authentication/exception/InvalidLoginCredentialsException.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/authentication/exception/InvalidLoginCredentialsException.java b/nifi-api/src/main/java/org/apache/nifi/authentication/exception/InvalidLoginCredentialsException.java
new file mode 100644
index 0000000..fbfa324
--- /dev/null
+++ b/nifi-api/src/main/java/org/apache/nifi/authentication/exception/InvalidLoginCredentialsException.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.authentication.exception;
+
+/**
+ * Represents the case when the identity could not be confirmed because the
+ * login credentials were invalid.
+ */
+public class InvalidLoginCredentialsException extends RuntimeException {
+
+ public InvalidLoginCredentialsException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public InvalidLoginCredentialsException(String message) {
+ super(message);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-api/src/main/java/org/apache/nifi/authorization/AuthorityProvider.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/authorization/AuthorityProvider.java b/nifi-api/src/main/java/org/apache/nifi/authorization/AuthorityProvider.java
index 723ec33..716216d 100644
--- a/nifi-api/src/main/java/org/apache/nifi/authorization/AuthorityProvider.java
+++ b/nifi-api/src/main/java/org/apache/nifi/authorization/AuthorityProvider.java
@@ -31,33 +31,33 @@ import org.apache.nifi.authorization.exception.UnknownIdentityException;
public interface AuthorityProvider {
/**
- * @param dn of the user
- * @return whether the user with the specified DN is known to this authority
+ * @param identity of the user. The identity may be a dn, an email, a username, or any string that identities the user.
+ * @return whether the user with the specified identity is known to this authority
* provider. It is not necessary for the user to have any authorities
*/
- boolean doesDnExist(String dn) throws AuthorityAccessException;
+ boolean doesDnExist(String identity) throws AuthorityAccessException;
/**
* Get the authorities for the specified user. If the specified user exists
* but does not have any authorities, an empty set should be returned.
*
- * @param dn of the user to lookup
+ * @param identity of the user. The identity may be a dn, an email, a username, or any string that identities the user.
* @return the authorities for the specified user. If the specified user
* exists but does not have any authorities, an empty set should be returned
* @throws UnknownIdentityException if identity is not known
* @throws AuthorityAccessException if unable to access authorities
*/
- Set<Authority> getAuthorities(String dn) throws UnknownIdentityException, AuthorityAccessException;
+ Set<Authority> getAuthorities(String identity) throws UnknownIdentityException, AuthorityAccessException;
/**
* Sets the specified authorities for the specified user.
*
- * @param dn the specified user
+ * @param identity of the user. The identity may be a dn, an email, a username, or any string that identities the user.
* @param authorities the new authorities for the user
* @throws UnknownIdentityException if identity is not known
* @throws AuthorityAccessException if unable to access authorities
*/
- void setAuthorities(String dn, Set<Authority> authorities) throws UnknownIdentityException, AuthorityAccessException;
+ void setAuthorities(String identity, Set<Authority> authorities) throws UnknownIdentityException, AuthorityAccessException;
/**
* Gets the users for the specified authority.
@@ -72,32 +72,32 @@ public interface AuthorityProvider {
* Revokes the specified user. Its up to the implementor to determine the
* semantics of revocation.
*
- * @param dn the dn of the user
+ * @param identity of the user. The identity may be a dn, an email, a username, or any string that identities the user.
* @throws UnknownIdentityException if the user is not known
* @throws AuthorityAccessException if unable to access the authorities
*/
- void revokeUser(String dn) throws UnknownIdentityException, AuthorityAccessException;
+ void revokeUser(String identity) throws UnknownIdentityException, AuthorityAccessException;
/**
* Add the specified user.
*
- * @param dn of the user
+ * @param identity of the user. The identity may be a dn, an email, a username, or any string that identities the user.
* @param group Optional
* @throws UnknownIdentityException if the user is not known
* @throws AuthorityAccessException if unable to access the authorities
*/
- void addUser(String dn, String group) throws IdentityAlreadyExistsException, AuthorityAccessException;
+ void addUser(String identity, String group) throws IdentityAlreadyExistsException, AuthorityAccessException;
/**
* Gets the group for the specified user. Return null if the user does not
* belong to a group.
*
- * @param dn the user
+ * @param identity of the user. The identity may be a dn, an email, a username, or any string that identities the user.
* @return the group of the given user
* @throws UnknownIdentityException if the user is not known
* @throws AuthorityAccessException if unable to access the authorities
*/
- String getGroupForUser(String dn) throws UnknownIdentityException, AuthorityAccessException;
+ String getGroupForUser(String identity) throws UnknownIdentityException, AuthorityAccessException;
/**
* Revokes all users for a specified group. Its up to the implementor to
@@ -112,21 +112,21 @@ public interface AuthorityProvider {
/**
* Adds the specified users to the specified group.
*
- * @param dn the set of users to add to the group
+ * @param identity of the user. The identity may be a dn, an email, a username, or any string that identities the user.
* @param group to add users to
* @throws UnknownIdentityException if the user is not known
* @throws AuthorityAccessException if unable to access the authorities
*/
- void setUsersGroup(Set<String> dn, String group) throws UnknownIdentityException, AuthorityAccessException;
+ void setUsersGroup(Set<String> identity, String group) throws UnknownIdentityException, AuthorityAccessException;
/**
* Ungroups the specified user.
*
- * @param dn of the user
+ * @param identity of the user. The identity may be a dn, an email, a username, or any string that identities the user.
* @throws UnknownIdentityException if the user is not known
* @throws AuthorityAccessException if unable to access the authorities
*/
- void ungroupUser(String dn) throws UnknownIdentityException, AuthorityAccessException;
+ void ungroupUser(String identity) throws UnknownIdentityException, AuthorityAccessException;
/**
* Ungroups the specified group. Since the semantics of revocation is up to
@@ -143,18 +143,18 @@ public interface AuthorityProvider {
* Determines whether the user in the specified dnChain should be able to
* download the content for the flowfile with the specified attributes.
*
- * The first dn in the chain is the end user that the request was issued on
- * behalf of. The subsequent dn's in the chain represent entities proxying
+ * The first identity in the chain is the end user that the request was issued on
+ * behalf of. The subsequent identities in the chain represent entities proxying
* the user's request with the last being the proxy that sent the current
* request.
*
- * @param dnChain of the user
+ * @param proxyChain proxy chain of user identities that for the download request
* @param attributes of the flowfile being requested
* @return the authorization result
* @throws UnknownIdentityException if the user is not known
* @throws AuthorityAccessException if unable to access the authorities
*/
- DownloadAuthorization authorizeDownload(List<String> dnChain, Map<String, String> attributes) throws UnknownIdentityException, AuthorityAccessException;
+ DownloadAuthorization authorizeDownload(List<String> proxyChain, Map<String, String> attributes) throws UnknownIdentityException, AuthorityAccessException;
/**
* Called immediately after instance creation for implementers to perform
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-api/src/main/java/org/apache/nifi/web/NiFiWebConfigurationContext.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/web/NiFiWebConfigurationContext.java b/nifi-api/src/main/java/org/apache/nifi/web/NiFiWebConfigurationContext.java
index ae32b10..39bea4f 100644
--- a/nifi-api/src/main/java/org/apache/nifi/web/NiFiWebConfigurationContext.java
+++ b/nifi-api/src/main/java/org/apache/nifi/web/NiFiWebConfigurationContext.java
@@ -54,7 +54,7 @@ public interface NiFiWebConfigurationContext {
void saveActions(NiFiWebRequestContext requestContext, Collection<ConfigurationAction> actions);
/**
- * @return the current user dn. Returns null if no user is found
+ * @return the current user identity. The value may be a DN, an email, a username, or any string that identities the user. Returns null if no user is found
*/
String getCurrentUserDn();
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-api/src/main/java/org/apache/nifi/web/NiFiWebContext.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/web/NiFiWebContext.java b/nifi-api/src/main/java/org/apache/nifi/web/NiFiWebContext.java
index 55e90e8..96261e5 100644
--- a/nifi-api/src/main/java/org/apache/nifi/web/NiFiWebContext.java
+++ b/nifi-api/src/main/java/org/apache/nifi/web/NiFiWebContext.java
@@ -51,7 +51,7 @@ public interface NiFiWebContext {
void saveActions(Collection<ProcessorConfigurationAction> actions);
/**
- * @return the current user dn. Returns null if no user is found
+ * @return the current user identity. It may be a dn, an email, a username, or any string that identities the user. Returns null if no user is found
*/
String getCurrentUserDn();
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-assembly/LICENSE
----------------------------------------------------------------------
diff --git a/nifi-assembly/LICENSE b/nifi-assembly/LICENSE
index 5c499e3..e21ecc9 100644
--- a/nifi-assembly/LICENSE
+++ b/nifi-assembly/LICENSE
@@ -374,6 +374,28 @@ For details see http://jqueryui.com
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+This product bundles 'jquery.base64.js' which is available under an MIT style license.
+
+ Copyright (c) 2013 Yannick Albert (http://yckart.com/)
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the "Software"),
+ to deal in the Software without restriction, including without limitation
+ the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ sell copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
This product bundles 'SlickGrid v2.2' which is available under an MIT style license.
Copyright (c) 2010 Michael Leibman, http://github.com/mleibman/slickgrid
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-assembly/pom.xml b/nifi-assembly/pom.xml
index 961349f..86aa8bc 100644
--- a/nifi-assembly/pom.xml
+++ b/nifi-assembly/pom.xml
@@ -174,6 +174,11 @@ language governing permissions and limitations under the License. -->
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-ldap-iaa-providers-nar</artifactId>
+ <type>nar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
<artifactId>nifi-dbcp-service-nar</artifactId>
<type>nar</type>
</dependency>
@@ -268,6 +273,7 @@ language governing permissions and limitations under the License. -->
<nifi.flow.configuration.file>./conf/flow.xml.gz</nifi.flow.configuration.file>
<nifi.flow.configuration.archive.dir>./conf/archive/</nifi.flow.configuration.archive.dir>
+ <nifi.login.identity.provider.configuration.file>./conf/login-identity-providers.xml</nifi.login.identity.provider.configuration.file>
<nifi.authority.provider.configuration.file>./conf/authority-providers.xml</nifi.authority.provider.configuration.file>
<nifi.templates.directory>./conf/templates</nifi.templates.directory>
<nifi.database.directory>./database_repository</nifi.database.directory>
@@ -353,8 +359,10 @@ language governing permissions and limitations under the License. -->
<nifi.security.authorizedUsers.file>./conf/authorized-users.xml</nifi.security.authorizedUsers.file>
<nifi.security.user.credential.cache.duration>24 hours</nifi.security.user.credential.cache.duration>
<nifi.security.user.authority.provider>file-provider</nifi.security.user.authority.provider>
+ <nifi.security.user.login.identity.provider></nifi.security.user.login.identity.provider>
<nifi.security.x509.principal.extractor />
<nifi.security.support.new.account.requests />
+ <nifi.security.anonymous.authorities>ROLE_MONITOR,ROLE_DFM,ROLE_ADMIN,ROLE_PROVENANCE,ROLE_NIFI</nifi.security.anonymous.authorities>
<nifi.security.ocsp.responder.url />
<nifi.security.ocsp.responder.certificate />
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java
index a51d9be..809dd09 100644
--- a/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java
+++ b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java
@@ -25,10 +25,14 @@ import java.nio.file.InvalidPathException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
+import java.util.Set;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -45,6 +49,7 @@ public class NiFiProperties extends Properties {
public static final String FLOW_CONFIGURATION_FILE = "nifi.flow.configuration.file";
public static final String FLOW_CONFIGURATION_ARCHIVE_FILE = "nifi.flow.configuration.archive.file";
public static final String AUTHORITY_PROVIDER_CONFIGURATION_FILE = "nifi.authority.provider.configuration.file";
+ public static final String LOGIN_IDENTITY_PROVIDER_CONFIGURATION_FILE = "nifi.login.identity.provider.configuration.file";
public static final String REPOSITORY_DATABASE_DIRECTORY = "nifi.database.directory";
public static final String RESTORE_DIRECTORY = "nifi.restore.directory";
public static final String VERSION = "nifi.version";
@@ -126,11 +131,12 @@ public class NiFiProperties extends Properties {
public static final String SECURITY_TRUSTSTORE_PASSWD = "nifi.security.truststorePasswd";
public static final String SECURITY_NEED_CLIENT_AUTH = "nifi.security.needClientAuth";
public static final String SECURITY_USER_AUTHORITY_PROVIDER = "nifi.security.user.authority.provider";
+ public static final String SECURITY_USER_LOGIN_IDENTITY_PROVIDER = "nifi.security.user.login.identity.provider";
public static final String SECURITY_CLUSTER_AUTHORITY_PROVIDER_PORT = "nifi.security.cluster.authority.provider.port";
public static final String SECURITY_CLUSTER_AUTHORITY_PROVIDER_THREADS = "nifi.security.cluster.authority.provider.threads";
public static final String SECURITY_USER_CREDENTIAL_CACHE_DURATION = "nifi.security.user.credential.cache.duration";
public static final String SECURITY_SUPPORT_NEW_ACCOUNT_REQUESTS = "nifi.security.support.new.account.requests";
- public static final String SECURITY_DEFAULT_USER_ROLES = "nifi.security.default.user.roles";
+ public static final String SECURITY_ANONYMOUS_AUTHORITIES = "nifi.security.anonymous.authorities";
public static final String SECURITY_OCSP_RESPONDER_URL = "nifi.security.ocsp.responder.url";
public static final String SECURITY_OCSP_RESPONDER_CERTIFICATE = "nifi.security.ocsp.responder.certificate";
@@ -187,6 +193,7 @@ public class NiFiProperties extends Properties {
public static final String DEFAULT_TITLE = "NiFi";
public static final Boolean DEFAULT_AUTO_RESUME_STATE = true;
public static final String DEFAULT_AUTHORITY_PROVIDER_CONFIGURATION_FILE = "conf/authority-providers.xml";
+ public static final String DEFAULT_LOGIN_IDENTITY_PROVIDER_CONFIGURATION_FILE = "conf/login-identity-providers.xml";
public static final String DEFAULT_USER_CREDENTIAL_CACHE_DURATION = "24 hours";
public static final Integer DEFAULT_REMOTE_INPUT_PORT = null;
public static final Path DEFAULT_TEMPLATE_DIRECTORY = Paths.get("conf", "templates");
@@ -234,8 +241,7 @@ public class NiFiProperties extends Properties {
}
/**
- * This is the method through which the NiFiProperties object should be
- * obtained.
+ * This is the method through which the NiFiProperties object should be obtained.
*
* @return the NiFiProperties object to use
* @throws RuntimeException if unable to load properties file
@@ -424,8 +430,7 @@ public class NiFiProperties extends Properties {
}
/**
- * Returns whether the processors should be started automatically when the
- * application loads.
+ * Returns whether the processors should be started automatically when the application loads.
*
* @return Whether to auto start the processors or not
*/
@@ -436,8 +441,7 @@ public class NiFiProperties extends Properties {
}
/**
- * Returns the number of partitions that should be used for the FlowFile
- * Repository
+ * Returns the number of partitions that should be used for the FlowFile Repository
*
* @return the number of partitions
*/
@@ -448,8 +452,7 @@ public class NiFiProperties extends Properties {
}
/**
- * Returns the number of milliseconds between FlowFileRepository
- * checkpointing
+ * Returns the number of milliseconds between FlowFileRepository checkpointing
*
* @return the number of milliseconds between checkpoint events
*/
@@ -483,6 +486,18 @@ public class NiFiProperties extends Properties {
}
/**
+ * @return the user authorities file
+ */
+ public File getLoginIdentityProviderConfiguraitonFile() {
+ final String value = getProperty(LOGIN_IDENTITY_PROVIDER_CONFIGURATION_FILE);
+ if (StringUtils.isBlank(value)) {
+ return new File(DEFAULT_LOGIN_IDENTITY_PROVIDER_CONFIGURATION_FILE);
+ } else {
+ return new File(value);
+ }
+ }
+
+ /**
* Will default to true unless the value is explicitly set to false.
*
* @return Whether client auth is required
@@ -510,6 +525,19 @@ public class NiFiProperties extends Properties {
return shouldSupport;
}
+ public Set<String> getAnonymousAuthorities() {
+ final Set<String> authorities;
+
+ final String rawAnonymousAuthorities = getProperty(SECURITY_ANONYMOUS_AUTHORITIES);
+ if (!StringUtils.isEmpty(rawAnonymousAuthorities)) {
+ authorities = new HashSet<>(Arrays.asList(rawAnonymousAuthorities.split(",")));
+ } else {
+ authorities = Collections.EMPTY_SET;
+ }
+
+ return authorities;
+ }
+
// getters for web properties //
public Integer getPort() {
Integer port = null;
@@ -851,8 +879,7 @@ public class NiFiProperties extends Properties {
}
/**
- * Returns the database repository path. It simply returns the value
- * configured. No directories will be created as a result of this operation.
+ * Returns the database repository path. It simply returns the value configured. No directories will be created as a result of this operation.
*
* @return database repository path
* @throws InvalidPathException If the configured path is invalid
@@ -862,8 +889,7 @@ public class NiFiProperties extends Properties {
}
/**
- * Returns the flow file repository path. It simply returns the value
- * configured. No directories will be created as a result of this operation.
+ * Returns the flow file repository path. It simply returns the value configured. No directories will be created as a result of this operation.
*
* @return database repository path
* @throws InvalidPathException If the configured path is invalid
@@ -873,10 +899,8 @@ public class NiFiProperties extends Properties {
}
/**
- * Returns the content repository paths. This method returns a mapping of
- * file repository name to file repository paths. It simply returns the
- * values configured. No directories will be created as a result of this
- * operation.
+ * Returns the content repository paths. This method returns a mapping of file repository name to file repository paths. It simply returns the values configured. No directories will be created as
+ * a result of this operation.
*
* @return file repositories paths
* @throws InvalidPathException If any of the configured paths are invalid
@@ -900,10 +924,8 @@ public class NiFiProperties extends Properties {
}
/**
- * Returns the provenance repository paths. This method returns a mapping of
- * file repository name to file repository paths. It simply returns the
- * values configured. No directories will be created as a result of this
- * operation.
+ * Returns the provenance repository paths. This method returns a mapping of file repository name to file repository paths. It simply returns the values configured. No directories will be created
+ * as a result of this operation.
*
* @return the name and paths of all provenance repository locations
*/
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/CertificateUtils.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/CertificateUtils.java b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/CertificateUtils.java
index 5126933..6236d8e 100644
--- a/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/CertificateUtils.java
+++ b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/CertificateUtils.java
@@ -34,8 +34,7 @@ public final class CertificateUtils {
private static final Logger logger = LoggerFactory.getLogger(CertificateUtils.class);
/**
- * Returns true if the given keystore can be loaded using the given keystore
- * type and password. Returns false otherwise.
+ * Returns true if the given keystore can be loaded using the given keystore type and password. Returns false otherwise.
*
* @param keystore the keystore to validate
* @param keystoreType the type of the keystore
@@ -77,40 +76,29 @@ public final class CertificateUtils {
}
/**
- * Extracts the username from the specified DN. If the username cannot be
- * extracted because the CN is in an unrecognized format, the entire CN is
- * returned. If the CN cannot be extracted because the DN is in an
- * unrecognized format, the entire DN is returned.
+ * Extracts the username from the specified DN. If the username cannot be extracted because the CN is in an unrecognized format, the entire CN is returned. If the CN cannot be extracted because
+ * the DN is in an unrecognized format, the entire DN is returned.
*
* @param dn the dn to extract the username from
* @return the exatracted username
*/
public static String extractUsername(String dn) {
String username = dn;
- String cn = "";
// ensure the dn is specified
if (StringUtils.isNotBlank(dn)) {
-
- // attempt to locate the cn
- if (dn.startsWith("CN=")) {
- cn = StringUtils.substringBetween(dn, "CN=", ",");
- } else if (dn.startsWith("/CN=")) {
- cn = StringUtils.substringBetween(dn, "CN=", "/");
- } else if (dn.startsWith("C=") || dn.startsWith("/C=")) {
- cn = StringUtils.substringAfter(dn, "CN=");
- } else if (dn.startsWith("/") && StringUtils.contains(dn, "CN=")) {
- cn = StringUtils.substringAfter(dn, "CN=");
- }
-
- // attempt to get the username from the cn
- if (StringUtils.isNotBlank(cn)) {
- if (cn.endsWith(")")) {
- username = StringUtils.substringBetween(cn, "(", ")");
- } else if (cn.contains(" ")) {
- username = StringUtils.substringAfterLast(cn, " ");
+ // determine the separate
+ final String separator = StringUtils.indexOfIgnoreCase(dn, "/cn=") > 0 ? "/" : ",";
+
+ // attempt to locate the cd
+ final String cnPattern = "cn=";
+ final int cnIndex = StringUtils.indexOfIgnoreCase(dn, cnPattern);
+ if (cnIndex >= 0) {
+ int separatorIndex = StringUtils.indexOf(dn, separator, cnIndex);
+ if (separatorIndex > 0) {
+ username = StringUtils.substring(dn, cnIndex + cnPattern.length(), separatorIndex);
} else {
- username = cn;
+ username = StringUtils.substring(dn, cnIndex + cnPattern.length());
}
}
}
@@ -119,9 +107,7 @@ public final class CertificateUtils {
}
/**
- * Returns a list of subject alternative names. Any name that is represented
- * as a String by X509Certificate.getSubjectAlternativeNames() is converted
- * to lowercase and returned.
+ * Returns a list of subject alternative names. Any name that is represented as a String by X509Certificate.getSubjectAlternativeNames() is converted to lowercase and returned.
*
* @param certificate a certificate
* @return a list of subject alternative names; list is never null
@@ -137,12 +123,9 @@ public final class CertificateUtils {
final List<String> result = new ArrayList<>();
for (final List<?> generalName : altNames) {
/**
- * generalName has the name type as the first element a String or
- * byte array for the second element. We return any general names
- * that are String types.
+ * generalName has the name type as the first element a String or byte array for the second element. We return any general names that are String types.
*
- * We don't inspect the numeric name type because some certificates
- * incorrectly put IPs and DNS names under the wrong name types.
+ * We don't inspect the numeric name type because some certificates incorrectly put IPs and DNS names under the wrong name types.
*/
final Object value = generalName.get(1);
if (value instanceof String) {
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/pom.xml
index aeaeda8..b117618 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/pom.xml
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/pom.xml
@@ -53,7 +53,6 @@
<excludes>**/authorization/generated/*.java,</excludes>
</configuration>
</plugin>
-
</plugins>
</build>
<dependencies>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/AuditDataSourceFactoryBean.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/AuditDataSourceFactoryBean.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/AuditDataSourceFactoryBean.java
index aeb2755..87cd420 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/AuditDataSourceFactoryBean.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/AuditDataSourceFactoryBean.java
@@ -45,8 +45,8 @@ public class AuditDataSourceFactoryBean implements FactoryBean {
// ------------
private static final String CREATE_ACTION_TABLE = "CREATE TABLE ACTION ("
+ "ID INT NOT NULL PRIMARY KEY AUTO_INCREMENT, "
- + "USER_DN VARCHAR2(255) NOT NULL, "
- + "USER_NAME VARCHAR2(100) NOT NULL, "
+ + "IDENTITY VARCHAR2(4096) NOT NULL, "
+ + "USER_NAME VARCHAR2(4096) NOT NULL, "
+ "SOURCE_ID VARCHAR2(100) NOT NULL, "
+ "SOURCE_NAME VARCHAR2(1000) NOT NULL, "
+ "SOURCE_TYPE VARCHAR2(1000) NOT NULL, "
@@ -107,6 +107,10 @@ public class AuditDataSourceFactoryBean implements FactoryBean {
+ "FOREIGN KEY (ACTION_ID) REFERENCES ACTION(ID)"
+ ")";
+ private static final String RENAME_DN_COLUMN = "ALTER TABLE ACTION ALTER COLUMN USER_DN RENAME TO IDENTITY";
+ private static final String RESIZE_IDENTITY_COLUMN = "ALTER TABLE ACTION MODIFY IDENTITY VARCHAR(4096)";
+ private static final String RESIZE_USER_NAME_COLUMN = "ALTER TABLE ACTION MODIFY USER_NAME VARCHAR(4096)";
+
private JdbcConnectionPool connectionPool;
private NiFiProperties properties;
@@ -148,15 +152,15 @@ public class AuditDataSourceFactoryBean implements FactoryBean {
connection = connectionPool.getConnection();
connection.setAutoCommit(false);
+ // create a statement for initializing the database
+ statement = connection.createStatement();
+
// determine if the tables need to be created
rs = connection.getMetaData().getTables(null, null, "ACTION", null);
if (!rs.next()) {
logger.info("Database not built for repository: " + databaseUrl + ". Building now...");
RepositoryUtils.closeQuietly(rs);
- // create a statement for initializing the database
- statement = connection.createStatement();
-
// action table
statement.execute(CREATE_ACTION_TABLE);
@@ -171,6 +175,15 @@ public class AuditDataSourceFactoryBean implements FactoryBean {
statement.execute(CREATE_PURGE_DETAILS_TABLE);
} else {
logger.info("Existing database found and connected to at: " + databaseUrl);
+ RepositoryUtils.closeQuietly(rs);
+
+ // check if the DN column exists to see if we need to transform the table
+ rs = connection.getMetaData().getColumns(null, null, "ACTION", "USER_DN");
+ if (rs.next()) {
+ statement.execute(RENAME_DN_COLUMN);
+ statement.execute(RESIZE_IDENTITY_COLUMN);
+ statement.execute(RESIZE_USER_NAME_COLUMN);
+ }
}
// commit any changes
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/KeyDataSourceFactoryBean.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/KeyDataSourceFactoryBean.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/KeyDataSourceFactoryBean.java
new file mode 100644
index 0000000..688aa5a
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/KeyDataSourceFactoryBean.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.admin;
+
+import java.io.File;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import org.apache.commons.lang3.StringUtils;
+import org.h2.jdbcx.JdbcConnectionPool;
+import org.apache.nifi.util.NiFiProperties;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.FactoryBean;
+
+/**
+ *
+ */
+public class KeyDataSourceFactoryBean implements FactoryBean {
+
+ private static final Logger logger = LoggerFactory.getLogger(KeyDataSourceFactoryBean.class);
+ private static final String NF_USERNAME_PASSWORD = "nf";
+ private static final int MAX_CONNECTIONS = 5;
+
+ // database file name
+ private static final String KEY_DATABASE_FILE_NAME = "nifi-key";
+
+ // ----------
+ // keys table
+ // ----------
+ private static final String CREATE_KEY_TABLE = "CREATE TABLE KEY ("
+ + "ID INT NOT NULL PRIMARY KEY AUTO_INCREMENT, "
+ + "IDENTITY VARCHAR2(4096) NOT NULL UNIQUE, "
+ + "KEY VARCHAR2(100) NOT NULL"
+ + ")";
+
+ private JdbcConnectionPool connectionPool;
+
+ private NiFiProperties properties;
+
+ @Override
+ public Object getObject() throws Exception {
+ if (connectionPool == null) {
+
+ // locate the repository directory
+ String repositoryDirectoryPath = properties.getProperty(NiFiProperties.REPOSITORY_DATABASE_DIRECTORY);
+
+ // ensure the repository directory is specified
+ if (repositoryDirectoryPath == null) {
+ throw new NullPointerException("Database directory must be specified.");
+ }
+
+ // create a handle to the repository directory
+ File repositoryDirectory = new File(repositoryDirectoryPath);
+
+ // get a handle to the database file
+ File databaseFile = new File(repositoryDirectory, KEY_DATABASE_FILE_NAME);
+
+ // format the database url
+ String databaseUrl = "jdbc:h2:" + databaseFile + ";AUTOCOMMIT=OFF;DB_CLOSE_ON_EXIT=FALSE;LOCK_MODE=3";
+ String databaseUrlAppend = properties.getProperty(NiFiProperties.H2_URL_APPEND);
+ if (StringUtils.isNotBlank(databaseUrlAppend)) {
+ databaseUrl += databaseUrlAppend;
+ }
+
+ // create the pool
+ connectionPool = JdbcConnectionPool.create(databaseUrl, NF_USERNAME_PASSWORD, NF_USERNAME_PASSWORD);
+ connectionPool.setMaxConnections(MAX_CONNECTIONS);
+
+ Connection connection = null;
+ ResultSet rs = null;
+ Statement statement = null;
+ try {
+ // get a connection
+ connection = connectionPool.getConnection();
+ connection.setAutoCommit(false);
+
+ // create a statement for initializing the database
+ statement = connection.createStatement();
+
+ // determine if the tables need to be created
+ rs = connection.getMetaData().getTables(null, null, "KEY", null);
+ if (!rs.next()) {
+ logger.info("Database not built for repository: " + databaseUrl + ". Building now...");
+ RepositoryUtils.closeQuietly(rs);
+
+ // action table
+ statement.execute(CREATE_KEY_TABLE);
+ } else {
+ logger.info("Existing database found and connected to at: " + databaseUrl);
+ }
+
+ // commit any changes
+ connection.commit();
+ } catch (SQLException sqle) {
+ RepositoryUtils.rollback(connection, logger);
+ throw sqle;
+ } finally {
+ RepositoryUtils.closeQuietly(rs);
+ RepositoryUtils.closeQuietly(statement);
+ RepositoryUtils.closeQuietly(connection);
+ }
+ }
+
+ return connectionPool;
+ }
+
+ @Override
+ public Class getObjectType() {
+ return JdbcConnectionPool.class;
+ }
+
+ @Override
+ public boolean isSingleton() {
+ return true;
+ }
+
+ public void setProperties(NiFiProperties properties) {
+ this.properties = properties;
+ }
+
+ /**
+ * Disposes resources.
+ */
+ public void shutdown() {
+
+ // shutdown the connection pool
+ if (connectionPool != null) {
+ try {
+ connectionPool.dispose();
+ } catch (Exception e) {
+ logger.warn("Unable to dispose of connection pool: " + e.getMessage());
+ if (logger.isDebugEnabled()) {
+ logger.warn(StringUtils.EMPTY, e);
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/UserDataSourceFactoryBean.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/UserDataSourceFactoryBean.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/UserDataSourceFactoryBean.java
index ebcf574..6d8566e 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/UserDataSourceFactoryBean.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/UserDataSourceFactoryBean.java
@@ -21,8 +21,11 @@ import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
+import java.util.HashSet;
+import java.util.Set;
import java.util.UUID;
import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.authorization.Authority;
import org.h2.jdbcx.JdbcConnectionPool;
import org.apache.nifi.user.NiFiUser;
import org.apache.nifi.util.NiFiProperties;
@@ -41,8 +44,8 @@ public class UserDataSourceFactoryBean implements FactoryBean {
private static final String CREATE_USER_TABLE = "CREATE TABLE USER ("
+ "ID VARCHAR2(100) NOT NULL PRIMARY KEY, "
- + "DN VARCHAR2(255) NOT NULL UNIQUE, "
- + "USER_NAME VARCHAR2(100) NOT NULL, "
+ + "IDENTITY VARCHAR2(4096) NOT NULL UNIQUE, "
+ + "USER_NAME VARCHAR2(4096) NOT NULL, "
+ "USER_GROUP VARCHAR2(100), "
+ "CREATION TIMESTAMP NOT NULL, "
+ "LAST_ACCESSED TIMESTAMP, "
@@ -60,57 +63,30 @@ public class UserDataSourceFactoryBean implements FactoryBean {
+ ")";
private static final String INSERT_ANONYMOUS_USER = "INSERT INTO USER ("
- + "ID, DN, USER_NAME, CREATION, LAST_VERIFIED, JUSTIFICATION, STATUS"
+ + "ID, IDENTITY, USER_NAME, CREATION, LAST_VERIFIED, JUSTIFICATION, STATUS"
+ ") VALUES ("
+ "'" + UUID.randomUUID().toString() + "', "
- + "'" + NiFiUser.ANONYMOUS_USER_DN + "', "
- + "'" + NiFiUser.ANONYMOUS_USER_DN + "', "
+ + "'" + NiFiUser.ANONYMOUS_USER_IDENTITY + "', "
+ + "'" + NiFiUser.ANONYMOUS_USER_IDENTITY + "', "
+ "NOW(), "
+ "NOW(), "
+ "'Anonymous user needs no justification', "
+ "'ACTIVE'"
+ ")";
- private static final String INSERT_ANONYMOUS_MONITOR_AUTHORITY = "INSERT INTO AUTHORITY ("
+ private static final String INSERT_ANONYMOUS_AUTHORITY = "INSERT INTO AUTHORITY ("
+ "USER_ID, ROLE"
+ ") VALUES ("
- + "(SELECT ID FROM USER WHERE DN = '" + NiFiUser.ANONYMOUS_USER_DN + "'), "
- + "'ROLE_MONITOR'"
+ + "(SELECT ID FROM USER WHERE IDENTITY = '" + NiFiUser.ANONYMOUS_USER_IDENTITY + "'), "
+ + "'%s'"
+ ")";
- private static final String INSERT_ANONYMOUS_DFM_AUTHORITY = "INSERT INTO AUTHORITY ("
- + "USER_ID, ROLE"
- + ") VALUES ("
- + "(SELECT ID FROM USER WHERE DN = '" + NiFiUser.ANONYMOUS_USER_DN + "'), "
- + "'ROLE_DFM'"
- + ")";
+ private static final String DELETE_ANONYMOUS_AUTHORITIES = "DELETE FROM AUTHORITY "
+ + "WHERE USER_ID = (SELECT ID FROM USER WHERE IDENTITY = '" + NiFiUser.ANONYMOUS_USER_IDENTITY + "')";
- private static final String INSERT_ANONYMOUS_ADMIN_AUTHORITY = "INSERT INTO AUTHORITY ("
- + "USER_ID, ROLE"
- + ") VALUES ("
- + "(SELECT ID FROM USER WHERE DN = '" + NiFiUser.ANONYMOUS_USER_DN + "'), "
- + "'ROLE_ADMIN'"
- + ")";
-
- private static final String INSERT_ANONYMOUS_NIFI_AUTHORITY = "INSERT INTO AUTHORITY ("
- + "USER_ID, ROLE"
- + ") VALUES ("
- + "(SELECT ID FROM USER WHERE DN = '" + NiFiUser.ANONYMOUS_USER_DN + "'), "
- + "'ROLE_NIFI'"
- + ")";
-
- private static final String INSERT_ANONYMOUS_PROVENANCE_AUTHORITY = "INSERT INTO AUTHORITY ("
- + "USER_ID, ROLE"
- + ") VALUES ("
- + "(SELECT ID FROM USER WHERE DN = '" + NiFiUser.ANONYMOUS_USER_DN + "'), "
- + "'ROLE_PROVENANCE'"
- + ")";
-
- private static final String SELECT_ANONYMOUS_PROVENANCE_AUTHORITY = "SELECT * FROM AUTHORITY "
- + "WHERE "
- + "USER_ID = (SELECT ID FROM USER WHERE DN = '" + NiFiUser.ANONYMOUS_USER_DN + "') "
- + "AND "
- + "ROLE = 'ROLE_PROVENANCE'";
+ private static final String RENAME_DN_COLUMN = "ALTER TABLE USER ALTER COLUMN DN RENAME TO IDENTITY";
+ private static final String RESIZE_IDENTITY_COLUMN = "ALTER TABLE USER MODIFY IDENTITY VARCHAR(4096)";
+ private static final String RESIZE_USER_NAME_COLUMN = "ALTER TABLE USER MODIFY USER_NAME VARCHAR(4096)";
private JdbcConnectionPool connectionPool;
@@ -128,6 +104,17 @@ public class UserDataSourceFactoryBean implements FactoryBean {
throw new NullPointerException("Database directory must be specified.");
}
+ // get the roles being granted to anonymous users
+ final Set<String> rawAnonymousAuthorities = new HashSet<>(properties.getAnonymousAuthorities());
+ final Set<Authority> anonymousAuthorities = Authority.convertRawAuthorities(rawAnonymousAuthorities);
+
+ // ensure every authorities was recognized
+ if (rawAnonymousAuthorities.size() != anonymousAuthorities.size()) {
+ final Set<String> validAuthorities = Authority.convertAuthorities(anonymousAuthorities);
+ rawAnonymousAuthorities.removeAll(validAuthorities);
+ throw new IllegalStateException("Invalid authorities specified: " + StringUtils.join(rawAnonymousAuthorities, ", "));
+ }
+
// create a handle to the repository directory
File repositoryDirectory = new File(repositoryDirectoryPath);
@@ -161,21 +148,25 @@ public class UserDataSourceFactoryBean implements FactoryBean {
// seed the anonymous user
statement.execute(INSERT_ANONYMOUS_USER);
- statement.execute(INSERT_ANONYMOUS_MONITOR_AUTHORITY);
- statement.execute(INSERT_ANONYMOUS_DFM_AUTHORITY);
- statement.execute(INSERT_ANONYMOUS_ADMIN_AUTHORITY);
- statement.execute(INSERT_ANONYMOUS_NIFI_AUTHORITY);
} else {
logger.info("Existing database found and connected to at: " + databaseUrl);
+ RepositoryUtils.closeQuietly(rs);
+
+ // if the DN column exists, transform the table
+ rs = connection.getMetaData().getColumns(null, null, "USER", "DN");
+ if (rs.next()) {
+ statement.execute(RENAME_DN_COLUMN);
+ statement.execute(RESIZE_IDENTITY_COLUMN);
+ statement.execute(RESIZE_USER_NAME_COLUMN);
+ }
+
+ // remove all authorities for the anonymous user
+ statement.execute(DELETE_ANONYMOUS_AUTHORITIES);
}
- // close the previous result set
- RepositoryUtils.closeQuietly(rs);
-
- // merge in the provenance role to handle existing databases
- rs = statement.executeQuery(SELECT_ANONYMOUS_PROVENANCE_AUTHORITY);
- if (!rs.next()) {
- statement.execute(INSERT_ANONYMOUS_PROVENANCE_AUTHORITY);
+ // add all authorities for the anonymous user
+ for (final Authority authority : anonymousAuthorities) {
+ statement.execute(String.format(INSERT_ANONYMOUS_AUTHORITY, authority.name()));
}
// commit any changes
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/DAOFactory.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/DAOFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/DAOFactory.java
index dee4ef9..eb7e3ce 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/DAOFactory.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/dao/DAOFactory.java
@@ -26,4 +26,6 @@ public interface DAOFactory {
ActionDAO getActionDAO();
AuthorityDAO getAuthorityDAO();
+
+ KeyDAO getKeyDAO();
}
[21/50] [abbrv] nifi git commit: NIFI-1191 Adding missing tags for
ConvertAvroToJSON
Posted by mc...@apache.org.
NIFI-1191 Adding missing tags for ConvertAvroToJSON
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/c541c82c
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/c541c82c
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/c541c82c
Branch: refs/heads/NIFI-655
Commit: c541c82c354a53774743f380e23898cbddd4d203
Parents: 69bce2c
Author: Aldrin Piri <al...@apache.org>
Authored: Wed Nov 18 16:38:15 2015 -0500
Committer: Aldrin Piri <al...@apache.org>
Committed: Wed Nov 18 16:38:15 2015 -0500
----------------------------------------------------------------------
.../java/org/apache/nifi/processors/avro/ConvertAvroToJSON.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/c541c82c/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/src/main/java/org/apache/nifi/processors/avro/ConvertAvroToJSON.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/src/main/java/org/apache/nifi/processors/avro/ConvertAvroToJSON.java b/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/src/main/java/org/apache/nifi/processors/avro/ConvertAvroToJSON.java
index ec5c320..8faaa4f 100644
--- a/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/src/main/java/org/apache/nifi/processors/avro/ConvertAvroToJSON.java
+++ b/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/src/main/java/org/apache/nifi/processors/avro/ConvertAvroToJSON.java
@@ -38,6 +38,7 @@ import org.apache.nifi.annotation.behavior.SideEffectFree;
import org.apache.nifi.annotation.behavior.SupportsBatching;
import org.apache.nifi.annotation.behavior.WritesAttribute;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.flowfile.attributes.CoreAttributes;
@@ -51,6 +52,7 @@ import org.apache.nifi.processor.io.StreamCallback;
@SideEffectFree
@SupportsBatching
+@Tags({"avro", "convert", "json"})
@InputRequirement(Requirement.INPUT_REQUIRED)
@CapabilityDescription("Converts a Binary Avro record into a JSON object. This processor provides a direct mapping of an Avro field to a JSON field, such "
+ "that the resulting JSON will have the same hierarchical structure as the Avro document. Note that the Avro schema information will be lost, as this "
[32/50] [abbrv] nifi git commit: NIFI-1196 Providing handling of
FETCH provenance events for their "unique" property, transit URI,
within the framework and UI.
Posted by mc...@apache.org.
NIFI-1196 Providing handling of FETCH provenance events for their "unique" property, transit URI, within the framework and UI.
Reviewed by Tony Kurc (tkurc@apache.org)
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/08d59e43
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/08d59e43
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/08d59e43
Branch: refs/heads/NIFI-655
Commit: 08d59e437462dfcec3e1f7347bcd6941ee47818a
Parents: 40dd8a0
Author: Aldrin Piri <al...@apache.org>
Authored: Thu Nov 19 02:52:01 2015 -0500
Committer: Aldrin Piri <al...@apache.org>
Committed: Thu Nov 19 17:42:15 2015 -0500
----------------------------------------------------------------------
.../java/org/apache/nifi/provenance/StandardLineageResult.java | 1 +
.../apache/nifi/provenance/StandardProvenanceEventRecord.java | 1 +
.../nifi/controller/repository/StandardProcessSession.java | 4 +++-
.../src/main/webapp/js/nf/provenance/nf-provenance-table.js | 5 +++++
.../java/org/apache/nifi/provenance/StandardRecordReader.java | 2 ++
.../java/org/apache/nifi/provenance/StandardRecordWriter.java | 2 ++
6 files changed, 14 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/08d59e43/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardLineageResult.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardLineageResult.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardLineageResult.java
index 63c53d0..cf16fc0 100644
--- a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardLineageResult.java
+++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardLineageResult.java
@@ -288,6 +288,7 @@ public class StandardLineageResult implements ComputeLineageResult {
}
break;
case RECEIVE:
+ case FETCH:
case CREATE: {
// for a receive event, we want to create a FlowFile Node that represents the FlowFile received
// and create an edge from the Receive Event to the FlowFile Node
http://git-wip-us.apache.org/repos/asf/nifi/blob/08d59e43/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardProvenanceEventRecord.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardProvenanceEventRecord.java b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardProvenanceEventRecord.java
index 4eb7001..b504b04 100644
--- a/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardProvenanceEventRecord.java
+++ b/nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardProvenanceEventRecord.java
@@ -728,6 +728,7 @@ public final class StandardProvenanceEventRecord implements ProvenanceEventRecor
}
break;
case RECEIVE:
+ case FETCH:
case SEND:
assertSet(transitUri, "Transit URI");
break;
http://git-wip-us.apache.org/repos/asf/nifi/blob/08d59e43/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
index 2ab90cd..d447ddd 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
@@ -457,6 +457,7 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
bytesSent += event.getFileSize();
break;
case RECEIVE:
+ case FETCH:
flowFilesReceived++;
bytesReceived += event.getFileSize();
break;
@@ -616,7 +617,8 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
if (registeredTypes.contains(ProvenanceEventType.CREATE)
|| registeredTypes.contains(ProvenanceEventType.FORK)
|| registeredTypes.contains(ProvenanceEventType.JOIN)
- || registeredTypes.contains(ProvenanceEventType.RECEIVE)) {
+ || registeredTypes.contains(ProvenanceEventType.RECEIVE)
+ || registeredTypes.contains(ProvenanceEventType.FETCH)) {
creationEventRegistered = true;
}
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/08d59e43/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/provenance/nf-provenance-table.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/provenance/nf-provenance-table.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/provenance/nf-provenance-table.js
index 1e5550c..faa6adc 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/provenance/nf-provenance-table.js
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/provenance/nf-provenance-table.js
@@ -1203,6 +1203,11 @@ nf.ProvenanceTable = (function () {
formatEventDetail('Relationship', event.relationship);
}
+ // conditionally show FETCH details
+ if (event.eventType === 'FETCH') {
+ formatEventDetail('Transit Uri', event.transitUri);
+ }
+
// conditionally show the cluster node identifier
if (nf.Common.isDefinedAndNotNull(event.clusterNodeId)) {
// save the cluster node id
http://git-wip-us.apache.org/repos/asf/nifi/blob/08d59e43/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/StandardRecordReader.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/StandardRecordReader.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/StandardRecordReader.java
index 42bc8e9..a9d7bf7 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/StandardRecordReader.java
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/StandardRecordReader.java
@@ -347,6 +347,8 @@ public class StandardRecordReader implements RecordReader {
} else if (eventType == ProvenanceEventType.RECEIVE) {
builder.setTransitUri(readNullableString(dis));
builder.setSourceSystemFlowFileIdentifier(readNullableString(dis));
+ } else if (eventType == ProvenanceEventType.FETCH) {
+ builder.setTransitUri(readNullableString(dis));
} else if (eventType == ProvenanceEventType.SEND) {
builder.setTransitUri(readNullableString(dis));
} else if (eventType == ProvenanceEventType.ADDINFO) {
http://git-wip-us.apache.org/repos/asf/nifi/blob/08d59e43/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/StandardRecordWriter.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/StandardRecordWriter.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/StandardRecordWriter.java
index ad3c6e2..981301e 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/StandardRecordWriter.java
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/StandardRecordWriter.java
@@ -235,6 +235,8 @@ public class StandardRecordWriter implements RecordWriter {
} else if (recordType == ProvenanceEventType.RECEIVE) {
writeNullableString(out, record.getTransitUri());
writeNullableString(out, record.getSourceSystemFlowFileIdentifier());
+ } else if (recordType == ProvenanceEventType.FETCH) {
+ writeNullableString(out, record.getTransitUri());
} else if (recordType == ProvenanceEventType.SEND) {
writeNullableString(out, record.getTransitUri());
} else if (recordType == ProvenanceEventType.ADDINFO) {
[15/50] [abbrv] nifi git commit: NIFI-1181: Ensure that a FlowFile's
uuid cannot be modified by processors
Posted by mc...@apache.org.
NIFI-1181: Ensure that a FlowFile's uuid cannot be modified by processors
Signed-off-by: Aldrin Piri <al...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/93be7533
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/93be7533
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/93be7533
Branch: refs/heads/NIFI-655
Commit: 93be753301559119205803182a49a550a5aa3486
Parents: 2b9b5e0
Author: Mark Payne <ma...@hotmail.com>
Authored: Wed Nov 18 09:04:22 2015 -0500
Committer: Aldrin Piri <al...@apache.org>
Committed: Wed Nov 18 10:35:37 2015 -0500
----------------------------------------------------------------------
.../repository/StandardFlowFileRecord.java | 13 +++++++
.../repository/StandardProcessSession.java | 26 ++++++-------
.../repository/TestStandardProcessSession.java | 40 ++++++++++++++++++++
3 files changed, 66 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/93be7533/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardFlowFileRecord.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardFlowFileRecord.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardFlowFileRecord.java
index 5474c7a..d3d8d40 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardFlowFileRecord.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardFlowFileRecord.java
@@ -247,6 +247,10 @@ public final class StandardFlowFileRecord implements FlowFile, FlowFileRecord {
public Builder removeAttributes(final String... keys) {
if (keys != null) {
for (final String key : keys) {
+ if (CoreAttributes.UUID.key().equals(key)) {
+ continue;
+ }
+
bAttributes.remove(key);
}
}
@@ -256,6 +260,10 @@ public final class StandardFlowFileRecord implements FlowFile, FlowFileRecord {
public Builder removeAttributes(final Set<String> keys) {
if (keys != null) {
for (final String key : keys) {
+ if (CoreAttributes.UUID.key().equals(key)) {
+ continue;
+ }
+
bAttributes.remove(key);
}
}
@@ -267,6 +275,11 @@ public final class StandardFlowFileRecord implements FlowFile, FlowFileRecord {
final Iterator<String> iterator = bAttributes.keySet().iterator();
while (iterator.hasNext()) {
final String key = iterator.next();
+
+ if (CoreAttributes.UUID.key().equals(key)) {
+ continue;
+ }
+
if (keyPattern.matcher(key).matches()) {
iterator.remove();
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/93be7533/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
index 3709972..2ab90cd 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
@@ -1410,14 +1410,18 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
validateRecordState(flowFile);
final StandardRepositoryRecord record = records.get(flowFile);
- final String originalUuid = flowFile.getAttribute(CoreAttributes.UUID.key());
+ final Map<String, String> updatedAttributes;
+ if (attributes.containsKey(CoreAttributes.UUID.key())) {
+ updatedAttributes = new HashMap<>(attributes);
+ updatedAttributes.remove(CoreAttributes.UUID.key());
+ } else {
+ updatedAttributes = attributes;
+ }
- final StandardFlowFileRecord.Builder ffBuilder = new StandardFlowFileRecord.Builder().fromFlowFile(record.getCurrent()).addAttributes(attributes);
- // Ignore the uuid attribute, if passed in
- ffBuilder.addAttribute(CoreAttributes.UUID.key(), originalUuid);
+ final StandardFlowFileRecord.Builder ffBuilder = new StandardFlowFileRecord.Builder().fromFlowFile(record.getCurrent()).addAttributes(updatedAttributes);
final FlowFileRecord newFile = ffBuilder.build();
- record.setWorking(newFile, attributes);
+ record.setWorking(newFile, updatedAttributes);
return newFile;
}
@@ -1443,19 +1447,15 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
return flowFile;
}
- final Set<String> keysToRemove;
- if (keys.contains(CoreAttributes.UUID.key())) {
- keysToRemove = new HashSet<>(keys);
- keysToRemove.remove(CoreAttributes.UUID.key());
- } else {
- keysToRemove = keys;
- }
-
final StandardRepositoryRecord record = records.get(flowFile);
final FlowFileRecord newFile = new StandardFlowFileRecord.Builder().fromFlowFile(record.getCurrent()).removeAttributes(keys).build();
final Map<String, String> updatedAttrs = new HashMap<>();
for (final String key : keys) {
+ if (CoreAttributes.UUID.key().equals(key)) {
+ continue;
+ }
+
updatedAttrs.put(key, null);
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/93be7533/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/TestStandardProcessSession.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/TestStandardProcessSession.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/TestStandardProcessSession.java
index 845257b..d549a00 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/TestStandardProcessSession.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/TestStandardProcessSession.java
@@ -37,13 +37,16 @@ import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.regex.Pattern;
import org.apache.nifi.connectable.Connectable;
import org.apache.nifi.connectable.ConnectableType;
@@ -58,6 +61,7 @@ import org.apache.nifi.controller.repository.claim.StandardContentClaim;
import org.apache.nifi.controller.repository.claim.StandardResourceClaim;
import org.apache.nifi.controller.repository.claim.StandardResourceClaimManager;
import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
import org.apache.nifi.groups.ProcessGroup;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.exception.FlowFileAccessException;
@@ -470,6 +474,42 @@ public class TestStandardProcessSession {
}
@Test
+ public void testUuidAttributeCannotBeUpdated() {
+ String originalUuid = "11111111-1111-1111-1111-111111111111";
+ final FlowFileRecord flowFileRecord1 = new StandardFlowFileRecord.Builder()
+ .id(1L)
+ .addAttribute("uuid", originalUuid)
+ .entryDate(System.currentTimeMillis())
+ .build();
+
+ flowFileQueue.put(flowFileRecord1);
+
+ FlowFile flowFile = session.get();
+ assertNotNull(flowFile);
+
+ final String uuid = CoreAttributes.UUID.key();
+ final String newUuid = "22222222-2222-2222-2222-222222222222";
+ flowFile = session.putAttribute(flowFile, uuid, newUuid);
+ assertEquals(originalUuid, flowFile.getAttribute(uuid));
+
+ final Map<String, String> uuidMap = new HashMap<>(1);
+ uuidMap.put(uuid, newUuid);
+
+ flowFile = session.putAllAttributes(flowFile, uuidMap);
+ assertEquals(originalUuid, flowFile.getAttribute(uuid));
+
+ flowFile = session.removeAllAttributes(flowFile, Pattern.compile("uuid"));
+ assertEquals(originalUuid, flowFile.getAttribute(uuid));
+
+ flowFile = session.removeAllAttributes(flowFile, Collections.singleton(uuid));
+ assertEquals(originalUuid, flowFile.getAttribute(uuid));
+
+ flowFile = session.removeAttribute(flowFile, uuid);
+ assertEquals(originalUuid, flowFile.getAttribute(uuid));
+
+ }
+
+ @Test
public void testUpdateAttributesThenJoin() throws IOException {
final FlowFileRecord flowFileRecord1 = new StandardFlowFileRecord.Builder()
.id(1L)
[47/50] [abbrv] nifi git commit: NIFI-655: - Refactoring web security
to use Spring Security Java Configuration. - Introducing security in Web UI
in order to get JWT.
Posted by mc...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/CreateUserActionTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/CreateUserActionTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/CreateUserActionTest.java
index 6486d32..e372781 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/CreateUserActionTest.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/CreateUserActionTest.java
@@ -40,8 +40,8 @@ public class CreateUserActionTest {
private final String USER_ID_2 = "2";
private final String USER_ID_3 = "3";
- private final String USER_DN_1 = "data access exception when creating user";
- private final String USER_DN_3 = "general create user case";
+ private final String USER_IDENTITY_1 = "data access exception when creating user";
+ private final String USER_IDENTITY_3 = "general create user case";
private DAOFactory daoFactory;
private UserDAO userDao;
@@ -57,9 +57,9 @@ public class CreateUserActionTest {
Object[] args = invocation.getArguments();
NiFiUser user = (NiFiUser) args[0];
- if (USER_DN_1.equals(user.getDn())) {
+ if (USER_IDENTITY_1.equals(user.getIdentity())) {
throw new DataAccessException();
- } else if (USER_DN_3.equals(user.getDn())) {
+ } else if (USER_IDENTITY_3.equals(user.getIdentity())) {
user.setId(USER_ID_3);
}
@@ -100,7 +100,7 @@ public class CreateUserActionTest {
@Test(expected = DataAccessException.class)
public void testExceptionCreatingUser() throws Exception {
NiFiUser user = new NiFiUser();
- user.setDn(USER_DN_1);
+ user.setIdentity(USER_IDENTITY_1);
CreateUserAction createUser = new CreateUserAction(user);
createUser.execute(daoFactory, null);
@@ -128,7 +128,7 @@ public class CreateUserActionTest {
@Test
public void testCreateUserAccount() throws Exception {
NiFiUser user = new NiFiUser();
- user.setDn(USER_DN_3);
+ user.setIdentity(USER_IDENTITY_3);
user.getAuthorities().addAll(EnumSet.of(Authority.ROLE_DFM, Authority.ROLE_ADMIN));
CreateUserAction createUser = new CreateUserAction(user);
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/DisableUserActionTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/DisableUserActionTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/DisableUserActionTest.java
index b0e1ac1..ac2ab29 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/DisableUserActionTest.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/DisableUserActionTest.java
@@ -40,8 +40,8 @@ public class DisableUserActionTest {
private static final String USER_ID_3 = "3";
private static final String USER_ID_4 = "4";
- private static final String USER_DN_3 = "authority access exception";
- private static final String USER_DN_4 = "general disable user case";
+ private static final String USER_IDENTITY_3 = "authority access exception";
+ private static final String USER_IDENTITY_4 = "general disable user case";
private DAOFactory daoFactory;
private UserDAO userDao;
@@ -66,11 +66,11 @@ public class DisableUserActionTest {
} else if (USER_ID_3.equals(id)) {
user = new NiFiUser();
user.setId(id);
- user.setDn(USER_DN_3);
+ user.setIdentity(USER_IDENTITY_3);
} else if (USER_ID_4.equals(id)) {
user = new NiFiUser();
user.setId(id);
- user.setDn(USER_DN_4);
+ user.setIdentity(USER_IDENTITY_4);
user.setStatus(AccountStatus.ACTIVE);
}
return user;
@@ -103,7 +103,7 @@ public class DisableUserActionTest {
Object[] args = invocation.getArguments();
String dn = (String) args[0];
- if (USER_DN_3.equals(dn)) {
+ if (USER_IDENTITY_3.equals(dn)) {
throw new AuthorityAccessException(StringUtils.EMPTY);
}
@@ -158,11 +158,11 @@ public class DisableUserActionTest {
// verify the user
Assert.assertEquals(USER_ID_4, user.getId());
- Assert.assertEquals(USER_DN_4, user.getDn());
+ Assert.assertEquals(USER_IDENTITY_4, user.getIdentity());
Assert.assertEquals(AccountStatus.DISABLED, user.getStatus());
// verify the interaction with the dao and provider
Mockito.verify(userDao, Mockito.times(1)).updateUser(user);
- Mockito.verify(authorityProvider, Mockito.times(1)).revokeUser(USER_DN_4);
+ Mockito.verify(authorityProvider, Mockito.times(1)).revokeUser(USER_IDENTITY_4);
}
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/RequestUserAccountActionTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/RequestUserAccountActionTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/RequestUserAccountActionTest.java
index 7707b2c..7bc863b 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/RequestUserAccountActionTest.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/RequestUserAccountActionTest.java
@@ -36,9 +36,9 @@ public class RequestUserAccountActionTest {
private static final String USER_ID_3 = "3";
- private static final String USER_DN_1 = "existing user account dn";
- private static final String USER_DN_2 = "data access exception";
- private static final String USER_DN_3 = "new account request";
+ private static final String USER_IDENTITY_1 = "existing user account";
+ private static final String USER_IDENTITY_2 = "data access exception";
+ private static final String USER_IDENTITY_3 = "new account request";
private DAOFactory daoFactory;
private UserDAO userDao;
@@ -54,7 +54,7 @@ public class RequestUserAccountActionTest {
String dn = (String) args[0];
NiFiUser user = null;
- if (USER_DN_1.equals(dn)) {
+ if (USER_IDENTITY_1.equals(dn)) {
user = new NiFiUser();
}
return user;
@@ -65,10 +65,10 @@ public class RequestUserAccountActionTest {
public Void answer(InvocationOnMock invocation) throws Throwable {
Object[] args = invocation.getArguments();
NiFiUser user = (NiFiUser) args[0];
- switch (user.getDn()) {
- case USER_DN_2:
+ switch (user.getIdentity()) {
+ case USER_IDENTITY_2:
throw new DataAccessException();
- case USER_DN_3:
+ case USER_IDENTITY_3:
user.setId(USER_ID_3);
break;
}
@@ -90,7 +90,7 @@ public class RequestUserAccountActionTest {
*/
@Test(expected = IllegalArgumentException.class)
public void testExistingAccount() throws Exception {
- RequestUserAccountAction requestUserAccount = new RequestUserAccountAction(USER_DN_1, StringUtils.EMPTY);
+ RequestUserAccountAction requestUserAccount = new RequestUserAccountAction(USER_IDENTITY_1, StringUtils.EMPTY);
requestUserAccount.execute(daoFactory, null);
}
@@ -102,7 +102,7 @@ public class RequestUserAccountActionTest {
*/
@Test(expected = DataAccessException.class)
public void testDataAccessException() throws Exception {
- RequestUserAccountAction requestUserAccount = new RequestUserAccountAction(USER_DN_2, StringUtils.EMPTY);
+ RequestUserAccountAction requestUserAccount = new RequestUserAccountAction(USER_IDENTITY_2, StringUtils.EMPTY);
requestUserAccount.execute(daoFactory, null);
}
@@ -113,12 +113,12 @@ public class RequestUserAccountActionTest {
*/
@Test
public void testRequestUserAccountAction() throws Exception {
- RequestUserAccountAction requestUserAccount = new RequestUserAccountAction(USER_DN_3, StringUtils.EMPTY);
+ RequestUserAccountAction requestUserAccount = new RequestUserAccountAction(USER_IDENTITY_3, StringUtils.EMPTY);
NiFiUser user = requestUserAccount.execute(daoFactory, null);
// verfiy the user
Assert.assertEquals(USER_ID_3, user.getId());
- Assert.assertEquals(USER_DN_3, user.getDn());
+ Assert.assertEquals(USER_IDENTITY_3, user.getIdentity());
Assert.assertEquals(AccountStatus.PENDING, user.getStatus());
// verify interaction with dao
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/SeedUserAccountsActionTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/SeedUserAccountsActionTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/SeedUserAccountsActionTest.java
index 652d992..58db56a 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/SeedUserAccountsActionTest.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/SeedUserAccountsActionTest.java
@@ -44,10 +44,10 @@ public class SeedUserAccountsActionTest {
private static final String USER_ID_3 = "3";
private static final String USER_ID_4 = "4";
- private static final String USER_DN_1 = "user dn 1 - active user - remove monitor and operator, add dfm";
- private static final String USER_DN_2 = "user dn 2 - active user - no action";
- private static final String USER_DN_3 = "user dn 3 - pending user - add operator";
- private static final String USER_DN_4 = "user dn 4 - new user - add monitor";
+ private static final String USER_IDENTITY_1 = "user 1 - active user - remove monitor and operator, add dfm";
+ private static final String USER_IDENTITY_2 = "user 2 - active user - no action";
+ private static final String USER_IDENTITY_3 = "user 3 - pending user - add operator";
+ private static final String USER_IDENTITY_4 = "user 4 - new user - add monitor";
private DAOFactory daoFactory;
private UserDAO userDao;
@@ -68,19 +68,19 @@ public class SeedUserAccountsActionTest {
if (USER_ID_1.equals(id)) {
user = new NiFiUser();
user.setId(USER_ID_1);
- user.setDn(USER_DN_1);
+ user.setIdentity(USER_IDENTITY_1);
user.getAuthorities().addAll(EnumSet.of(Authority.ROLE_MONITOR));
user.setStatus(AccountStatus.ACTIVE);
} else if (USER_ID_2.equals(id)) {
user = new NiFiUser();
user.setId(USER_ID_2);
- user.setDn(USER_DN_2);
+ user.setIdentity(USER_IDENTITY_2);
user.getAuthorities().addAll(EnumSet.of(Authority.ROLE_ADMIN));
user.setStatus(AccountStatus.ACTIVE);
} else if (USER_ID_3.equals(id)) {
user = new NiFiUser();
user.setId(USER_ID_3);
- user.setDn(USER_DN_3);
+ user.setIdentity(USER_IDENTITY_3);
user.setStatus(AccountStatus.PENDING);
}
return user;
@@ -93,22 +93,22 @@ public class SeedUserAccountsActionTest {
String dn = (String) args[0];
NiFiUser user = null;
- if (USER_DN_1.equals(dn)) {
+ if (USER_IDENTITY_1.equals(dn)) {
user = new NiFiUser();
user.setId(USER_ID_1);
- user.setDn(USER_DN_1);
+ user.setIdentity(USER_IDENTITY_1);
user.getAuthorities().addAll(EnumSet.of(Authority.ROLE_MONITOR));
user.setStatus(AccountStatus.ACTIVE);
- } else if (USER_DN_2.equals(dn)) {
+ } else if (USER_IDENTITY_2.equals(dn)) {
user = new NiFiUser();
user.setId(USER_ID_2);
- user.setDn(USER_DN_2);
+ user.setIdentity(USER_IDENTITY_2);
user.getAuthorities().addAll(EnumSet.of(Authority.ROLE_ADMIN));
user.setStatus(AccountStatus.ACTIVE);
- } else if (USER_DN_3.equals(dn)) {
+ } else if (USER_IDENTITY_3.equals(dn)) {
user = new NiFiUser();
user.setId(USER_ID_3);
- user.setDn(USER_DN_3);
+ user.setIdentity(USER_IDENTITY_3);
user.setStatus(AccountStatus.PENDING);
}
return user;
@@ -120,7 +120,7 @@ public class SeedUserAccountsActionTest {
Object[] args = invocation.getArguments();
NiFiUser user = (NiFiUser) args[0];
- if (USER_DN_4.equals(user.getDn())) {
+ if (USER_IDENTITY_4.equals(user.getIdentity())) {
user.setId(USER_ID_4);
}
@@ -141,13 +141,13 @@ public class SeedUserAccountsActionTest {
Set<String> users = new HashSet<>();
if (Authority.ROLE_DFM.equals(role)) {
- users.add(USER_DN_1);
+ users.add(USER_IDENTITY_1);
} else if (Authority.ROLE_ADMIN.equals(role)) {
- users.add(USER_DN_2);
+ users.add(USER_IDENTITY_2);
} else if (Authority.ROLE_PROXY.equals(role)) {
- users.add(USER_DN_3);
+ users.add(USER_IDENTITY_3);
} else if (Authority.ROLE_MONITOR.equals(role)) {
- users.add(USER_DN_4);
+ users.add(USER_IDENTITY_4);
}
return users;
}
@@ -160,16 +160,16 @@ public class SeedUserAccountsActionTest {
Set<Authority> authorities = EnumSet.noneOf(Authority.class);
switch (dn) {
- case USER_DN_1:
+ case USER_IDENTITY_1:
authorities.add(Authority.ROLE_DFM);
break;
- case USER_DN_2:
+ case USER_IDENTITY_2:
authorities.add(Authority.ROLE_ADMIN);
break;
- case USER_DN_3:
+ case USER_IDENTITY_3:
authorities.add(Authority.ROLE_PROXY);
break;
- case USER_DN_4:
+ case USER_IDENTITY_4:
authorities.add(Authority.ROLE_MONITOR);
break;
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/SetUserAuthoritiesActionTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/SetUserAuthoritiesActionTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/SetUserAuthoritiesActionTest.java
index 22504f7..5effdbb 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/SetUserAuthoritiesActionTest.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/test/java/org/apache/nifi/admin/service/action/SetUserAuthoritiesActionTest.java
@@ -46,8 +46,8 @@ public class SetUserAuthoritiesActionTest {
private static final String USER_ID_2 = "2";
private static final String USER_ID_3 = "3";
- private static final String USER_DN_2 = "user dn 2";
- private static final String USER_DN_3 = "user dn 3";
+ private static final String USER_IDENTITY_2 = "user 2";
+ private static final String USER_IDENTITY_3 = "user 3";
private DAOFactory daoFactory;
private UserDAO userDao;
@@ -70,11 +70,11 @@ public class SetUserAuthoritiesActionTest {
} else if (USER_ID_2.equals(id)) {
user = new NiFiUser();
user.setId(USER_ID_2);
- user.setDn(USER_DN_2);
+ user.setIdentity(USER_IDENTITY_2);
} else if (USER_ID_3.equals(id)) {
user = new NiFiUser();
user.setId(USER_ID_3);
- user.setDn(USER_DN_3);
+ user.setIdentity(USER_IDENTITY_3);
user.getAuthorities().addAll(EnumSet.of(Authority.ROLE_MONITOR));
user.setStatus(AccountStatus.ACTIVE);
}
@@ -88,10 +88,10 @@ public class SetUserAuthoritiesActionTest {
String dn = (String) args[0];
NiFiUser user = null;
- if (USER_DN_3.equals(dn)) {
+ if (USER_IDENTITY_3.equals(dn)) {
user = new NiFiUser();
user.setId(USER_ID_3);
- user.setDn(USER_DN_3);
+ user.setIdentity(USER_IDENTITY_3);
user.getAuthorities().addAll(EnumSet.of(Authority.ROLE_MONITOR));
user.setStatus(AccountStatus.ACTIVE);
}
@@ -148,7 +148,7 @@ public class SetUserAuthoritiesActionTest {
String dn = (String) args[0];
Set<Authority> authorities = EnumSet.noneOf(Authority.class);
- if (USER_DN_3.equals(dn)) {
+ if (USER_IDENTITY_3.equals(dn)) {
authorities.add(Authority.ROLE_DFM);
}
@@ -162,7 +162,7 @@ public class SetUserAuthoritiesActionTest {
String dn = (String) args[0];
Set<Authority> authorites = (Set<Authority>) args[1];
- if (USER_DN_2.equals(dn)) {
+ if (USER_IDENTITY_2.equals(dn)) {
throw new AuthorityAccessException(StringUtils.EMPTY);
}
@@ -218,6 +218,6 @@ public class SetUserAuthoritiesActionTest {
Set<Authority> authoritiesAddedToProvider = EnumSet.of(Authority.ROLE_ADMIN);
// verify interaction with provider
- Mockito.verify(authorityProvider, Mockito.times(1)).setAuthorities(USER_DN_3, authoritiesAddedToProvider);
+ Mockito.verify(authorityProvider, Mockito.times(1)).setAuthorities(USER_IDENTITY_3, authoritiesAddedToProvider);
}
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/AccessConfigurationDTO.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/AccessConfigurationDTO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/AccessConfigurationDTO.java
new file mode 100644
index 0000000..d9719b3
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/AccessConfigurationDTO.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.api.dto;
+
+import com.wordnik.swagger.annotations.ApiModelProperty;
+import javax.xml.bind.annotation.XmlType;
+
+/**
+ * Details for the access configuration.
+ */
+@XmlType(name = "accessConfig")
+public class AccessConfigurationDTO {
+
+ private Boolean supportsLogin;
+ private Boolean supportsAnonymous;
+
+ /**
+ * @return Indicates whether or not this NiFi supports user login.
+ */
+ @ApiModelProperty(
+ value = "Indicates whether or not this NiFi supports user login.",
+ readOnly = true
+ )
+ public Boolean getSupportsLogin() {
+ return supportsLogin;
+ }
+
+ public void setSupportsLogin(Boolean supportsLogin) {
+ this.supportsLogin = supportsLogin;
+ }
+
+ /**
+ * @return Indicates whether or not this NiFi supports anonymous access.
+ */
+ @ApiModelProperty(
+ value = "Indicates whether or not this NiFi supports anonymous.",
+ readOnly = true
+ )
+ public Boolean getSupportsAnonymous() {
+ return supportsAnonymous;
+ }
+
+ public void setSupportsAnonymous(Boolean supportsAnonymous) {
+ this.supportsAnonymous = supportsAnonymous;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/AccessStatusDTO.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/AccessStatusDTO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/AccessStatusDTO.java
new file mode 100644
index 0000000..712da0e
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/AccessStatusDTO.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.api.dto;
+
+import com.wordnik.swagger.annotations.ApiModelProperty;
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * A serialized representation of this class can be placed in the entity body of a response to the API. This particular entity holds the users access status.
+ */
+@XmlRootElement(name = "accessStatus")
+public class AccessStatusDTO {
+
+ public static enum Status {
+
+ UNKNOWN,
+ UNREGISTERED,
+ NOT_ACTIVE,
+ ACTIVE
+ }
+
+ private String identity;
+ private String username;
+ private String status;
+ private String message;
+
+ /**
+ * @return the user identity
+ */
+ @ApiModelProperty(
+ value = "The user identity.",
+ readOnly = true
+ )
+ public String getIdentity() {
+ return identity;
+ }
+
+ public void setIdentity(String identity) {
+ this.identity = identity;
+ }
+
+ /**
+ * @return the username
+ */
+ @ApiModelProperty(
+ value = "The username.",
+ readOnly = true
+ )
+ public String getUsername() {
+ return username;
+ }
+
+ public void setUsername(String username) {
+ this.username = username;
+ }
+
+ /**
+ * @return the user access status
+ */
+ @ApiModelProperty(
+ value = "The user access status.",
+ readOnly = true
+ )
+ public String getStatus() {
+ return status;
+ }
+
+ public void setStatus(String status) {
+ this.status = status;
+ }
+
+ /**
+ * @return additional details about the user access status
+ */
+ @ApiModelProperty(
+ value = "Additional details about the user access status.",
+ readOnly = true
+ )
+ public String getMessage() {
+ return message;
+ }
+
+ public void setMessage(String message) {
+ this.message = message;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/AccessConfigurationEntity.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/AccessConfigurationEntity.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/AccessConfigurationEntity.java
new file mode 100644
index 0000000..3af0e49
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/AccessConfigurationEntity.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.api.entity;
+
+import javax.xml.bind.annotation.XmlRootElement;
+import org.apache.nifi.web.api.dto.AccessConfigurationDTO;
+
+/**
+ * A serialized representation of this class can be placed in the entity body of a request or response to or from the API. This particular entity holds a reference to a AccessConfigurationDTO.
+ */
+@XmlRootElement(name = "accessConfigurationEntity")
+public class AccessConfigurationEntity extends Entity {
+
+ private AccessConfigurationDTO config;
+
+ /**
+ * The AccessConfigurationDTO that is being serialized.
+ *
+ * @return The AccessConfigurationDTO object
+ */
+ public AccessConfigurationDTO getConfig() {
+ return config;
+ }
+
+ public void setConfig(AccessConfigurationDTO config) {
+ this.config = config;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/AccessStatusEntity.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/AccessStatusEntity.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/AccessStatusEntity.java
new file mode 100644
index 0000000..f19a268
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/AccessStatusEntity.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.api.entity;
+
+import javax.xml.bind.annotation.XmlRootElement;
+import org.apache.nifi.web.api.dto.AccessStatusDTO;
+
+/**
+ * A serialized representation of this class can be placed in the entity body of a request or response to or from the API. This particular entity holds a reference to a AccessStatusDTO.
+ */
+@XmlRootElement(name = "accessStatusEntity")
+public class AccessStatusEntity extends Entity {
+
+ private AccessStatusDTO accessStatus;
+
+ /**
+ * The AccessStatusDTO that is being serialized.
+ *
+ * @return The AccessStatusDTO object
+ */
+ public AccessStatusDTO getAccessStatus() {
+ return accessStatus;
+ }
+
+ public void setAccessStatus(AccessStatusDTO accessStatus) {
+ this.accessStatus = accessStatus;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/IdentityEntity.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/IdentityEntity.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/IdentityEntity.java
new file mode 100644
index 0000000..02991c7
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/IdentityEntity.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.api.entity;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * A serialized representation of this class can be placed in the entity body of a response to the API. This particular entity holds the users identity.
+ */
+@XmlRootElement(name = "identityEntity")
+public class IdentityEntity extends Entity {
+
+ private String userId;
+ private String identity;
+
+ /**
+ * @return current user id
+ */
+ public String getUserId() {
+ return userId;
+ }
+
+ public void setUserId(String userId) {
+ this.userId = userId;
+ }
+
+ /**
+ * @return the user identity being serialized
+ */
+ public String getIdentity() {
+ return identity;
+ }
+
+ public void setIdentity(String identity) {
+ this.identity = identity;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/main/java/org/apache/nifi/nar/ExtensionManager.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/main/java/org/apache/nifi/nar/ExtensionManager.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/main/java/org/apache/nifi/nar/ExtensionManager.java
index 9bbc3a3..db0b35e 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/main/java/org/apache/nifi/nar/ExtensionManager.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/main/java/org/apache/nifi/nar/ExtensionManager.java
@@ -22,6 +22,7 @@ import java.util.HashSet;
import java.util.Map;
import java.util.ServiceLoader;
import java.util.Set;
+import org.apache.nifi.authentication.LoginIdentityProvider;
import org.apache.nifi.authorization.AuthorityProvider;
import org.apache.nifi.controller.ControllerService;
@@ -38,9 +39,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * Scans through the classpath to load all FlowFileProcessors,
- * FlowFileComparators, and ReportingTasks using the service provider API and
- * running through all classloaders (root, NARs).
+ * Scans through the classpath to load all FlowFileProcessors, FlowFileComparators, and ReportingTasks using the service provider API and running through all classloaders (root, NARs).
*
* @ThreadSafe - is immutable
*/
@@ -60,6 +59,7 @@ public class ExtensionManager {
definitionMap.put(ReportingTask.class, new HashSet<Class>());
definitionMap.put(ControllerService.class, new HashSet<Class>());
definitionMap.put(AuthorityProvider.class, new HashSet<Class>());
+ definitionMap.put(LoginIdentityProvider.class, new HashSet<Class>());
definitionMap.put(ProvenanceEventRepository.class, new HashSet<Class>());
definitionMap.put(ComponentStatusRepository.class, new HashSet<Class>());
definitionMap.put(FlowFileRepository.class, new HashSet<Class>());
@@ -68,9 +68,7 @@ public class ExtensionManager {
}
/**
- * Loads all FlowFileProcessor, FlowFileComparator, ReportingTask class
- * types that can be found on the bootstrap classloader and by creating
- * classloaders for all NARs found within the classpath.
+ * Loads all FlowFileProcessor, FlowFileComparator, ReportingTask class types that can be found on the bootstrap classloader and by creating classloaders for all NARs found within the classpath.
*/
public static void discoverExtensions() {
final ClassLoader systemClassLoader = ClassLoader.getSystemClassLoader();
@@ -113,8 +111,7 @@ public class ExtensionManager {
}
/**
- * Registers extension for the specified type from the specified
- * ClassLoader.
+ * Registers extension for the specified type from the specified ClassLoader.
*
* @param type the extension type
* @param classloaderMap mapping of classname to classloader
@@ -152,9 +149,7 @@ public class ExtensionManager {
}
/**
- * Determines the effective classloader for classes of the given type. If
- * returns null it indicates the given type is not known or was not
- * detected.
+ * Determines the effective classloader for classes of the given type. If returns null it indicates the given type is not known or was not detected.
*
* @param classType to lookup the classloader of
* @return String of fully qualified class name; null if not a detected type
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java
index 9471ba6..9e9bd03 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java
@@ -23,6 +23,7 @@ import java.net.URLClassLoader;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.List;
+import org.apache.nifi.authentication.LoginIdentityProvider;
import org.apache.nifi.authorization.AuthorityProvider;
import org.apache.nifi.components.Validator;
@@ -58,6 +59,7 @@ public class NarThreadContextClassLoader extends URLClassLoader {
narSpecificClasses.add(StreamCallback.class);
narSpecificClasses.add(ControllerService.class);
narSpecificClasses.add(AuthorityProvider.class);
+ narSpecificClasses.add(LoginIdentityProvider.class);
narSpecificClasses.add(ProvenanceEventRepository.class);
narSpecificClasses.add(ComponentStatusRepository.class);
narSpecificClasses.add(FlowFileRepository.class);
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/login-identity-providers.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/login-identity-providers.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/login-identity-providers.xml
new file mode 100644
index 0000000..9868b9d
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/login-identity-providers.xml
@@ -0,0 +1,92 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!--
+ This file lists the login identity providers to use when running securely. In order
+ to use a specific provider it must be configured here and it's identifier
+ must be specified in the nifi.properties file.
+-->
+<loginIdentityProviders>
+ <!--
+ Identity Provider for users logging in with username/password against an LDAP server.
+
+ 'Authentication Strategy' - How the connection to the LDAP server is authenticated. Possible
+ values are ANONYMOUS, SIMPLE, or START_TLS.
+
+ 'Manager DN' - The DN of the manager that is used to bind to the LDAP server to search for users.
+ 'Manager Password' - The password of the manager that is used to bind to the LDAP server to
+ search for users.
+
+ 'TLS - Keystore' - Path to the Keystore that is used when connecting to LDAP using START_TLS.
+ 'TLS - Keystore Password' - Password for the Keystore that is used when connecting to LDAP
+ using START_TLS.
+ 'TLS - Keystore Type' - Type of the Keystore that is used when connecting to LDAP using
+ START_TLS (i.e. JKS or PKCS12).
+ 'TLS - Truststore' - Path to the Truststore that is used when connecting to LDAP using START_TLS.
+ 'TLS - Truststore Password' - Password for the Truststore that is used when connecting to
+ LDAP using START_TLS.
+ 'TLS - Truststore Type' - Type of the Truststore that is used when connecting to LDAP using
+ START_TLS (i.e. JKS or PKCS12).
+ 'TLS - Client Auth' - Client authentication policy when connecting to LDAP using START_TLS.
+ Possible values are REQUIRED, WANT, NONE.
+ 'TLS - Protocol' - Protocol to use when connecting to LDAP using START_TLS. (i.e. TLS,
+ TLSv1.1, TLSv1.2, etc).
+ 'TLS - Shutdown Gracefully' - Specifies whether the TLS should be shut down gracefully
+ before the target context is closed. Defaults to false.
+
+ 'Referral Strategy' - Strategy for handling referrals. Possible values are FOLLOW, IGNORE, THROW.
+ 'Connect Timeout' - Duration of connect timeout. (i.e. 10 secs).
+ 'Read Timeout' - Duration of read timeout. (i.e. 10 secs).
+
+ 'Url' - Url of the LDAP servier (i.e. ldap://<hostname>:<port>).
+ 'User Search Base' - Base DN for searching for users (i.e. CN=Users,DC=example,DC=com).
+ 'User Search Filter' - Filter for searching for users against the 'User Search Base'.
+ (i.e. sAMAccountName={0}). The user specified name is inserted into '{0}'.
+
+ 'Authentication Expiration' - The duration of how long the user authentication is valid
+ for. If the user never logs out, they will be required to log back in following
+ this duration.
+ -->
+ <!-- To enable the ldap-provider remove 2 lines. This is 1 of 2.
+ <provider>
+ <identifier>ldap-provider</identifier>
+ <class>org.apache.nifi.ldap.LdapProvider</class>
+ <property name="Authentication Strategy">START_TLS</property>
+
+ <property name="Manager DN"></property>
+ <property name="Manager Password"></property>
+
+ <property name="TLS - Keystore"></property>
+ <property name="TLS - Keystore Password"></property>
+ <property name="TLS - Keystore Type"></property>
+ <property name="TLS - Truststore"></property>
+ <property name="TLS - Truststore Password"></property>
+ <property name="TLS - Truststore Type"></property>
+ <property name="TLS - Client Auth"></property>
+ <property name="TLS - Protocol"></property>
+ <property name="TLS - Shutdown Gracefully"></property>
+
+ <property name="Referral Strategy">FOLLOW</property>
+ <property name="Connect Timeout">10 secs</property>
+ <property name="Read Timeout">10 secs</property>
+
+ <property name="Url"></property>
+ <property name="User Search Base"></property>
+ <property name="User Search Filter"></property>
+
+ <property name="Expiration Duration">12 hours</property>
+ </provider>
+ To enable the ldap-provider remove 2 lines. This is 2 of 2. -->
+</loginIdentityProviders>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties
index 54b5283..b25d05a 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties
@@ -25,6 +25,7 @@ nifi.administrative.yield.duration=${nifi.administrative.yield.duration}
nifi.bored.yield.duration=${nifi.bored.yield.duration}
nifi.authority.provider.configuration.file=${nifi.authority.provider.configuration.file}
+nifi.login.identity.provider.configuration.file=${nifi.login.identity.provider.configuration.file}
nifi.templates.directory=${nifi.templates.directory}
nifi.ui.banner.text=${nifi.ui.banner.text}
nifi.ui.autorefresh.interval=${nifi.ui.autorefresh.interval}
@@ -124,7 +125,9 @@ nifi.security.truststorePasswd=${nifi.security.truststorePasswd}
nifi.security.needClientAuth=${nifi.security.needClientAuth}
nifi.security.user.credential.cache.duration=${nifi.security.user.credential.cache.duration}
nifi.security.user.authority.provider=${nifi.security.user.authority.provider}
+nifi.security.user.login.identity.provider=${nifi.security.user.login.identity.provider}
nifi.security.support.new.account.requests=${nifi.security.support.new.account.requests}
+nifi.security.anonymous.authorities=${nifi.security.anonymous.authorities}
nifi.security.ocsp.responder.url=${nifi.security.ocsp.responder.url}
nifi.security.ocsp.responder.certificate=${nifi.security.ocsp.responder.certificate}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/JettyServer.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/JettyServer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/JettyServer.java
index ecfe2c0..d1bd5c8 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/JettyServer.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/JettyServer.java
@@ -615,8 +615,12 @@ public class JettyServer implements NiFiServer {
private SslContextFactory createSslContextFactory() {
final SslContextFactory contextFactory = new SslContextFactory();
- // need client auth
- contextFactory.setNeedClientAuth(props.getNeedClientAuth());
+ // require client auth when not supporting login or anonymous access
+ if (StringUtils.isBlank(props.getProperty(NiFiProperties.SECURITY_USER_LOGIN_IDENTITY_PROVIDER)) && props.getAnonymousAuthorities().isEmpty()) {
+ contextFactory.setNeedClientAuth(true);
+ } else {
+ contextFactory.setWantClientAuth(true);
+ }
/* below code sets JSSE system properties when values are provided */
// keystore properties
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ControllerAuditor.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ControllerAuditor.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ControllerAuditor.java
index cede675..4357633 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ControllerAuditor.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ControllerAuditor.java
@@ -78,7 +78,7 @@ public class ControllerAuditor extends NiFiAuditor {
// create the config action
FlowChangeAction configAction = new FlowChangeAction();
- configAction.setUserIdentity(user.getDn());
+ configAction.setUserIdentity(user.getIdentity());
configAction.setUserName(user.getUserName());
configAction.setOperation(Operation.Configure);
configAction.setTimestamp(new Date());
@@ -131,7 +131,7 @@ public class ControllerAuditor extends NiFiAuditor {
// create the config action
FlowChangeAction configAction = new FlowChangeAction();
- configAction.setUserIdentity(user.getDn());
+ configAction.setUserIdentity(user.getIdentity());
configAction.setUserName(user.getUserName());
configAction.setOperation(Operation.Configure);
configAction.setTimestamp(new Date());
@@ -184,7 +184,7 @@ public class ControllerAuditor extends NiFiAuditor {
// create the config action
FlowChangeAction configAction = new FlowChangeAction();
- configAction.setUserIdentity(user.getDn());
+ configAction.setUserIdentity(user.getIdentity());
configAction.setUserName(user.getUserName());
configAction.setOperation(Operation.Configure);
configAction.setTimestamp(new Date());
@@ -237,7 +237,7 @@ public class ControllerAuditor extends NiFiAuditor {
// create the config action
FlowChangeAction configAction = new FlowChangeAction();
- configAction.setUserIdentity(user.getDn());
+ configAction.setUserIdentity(user.getIdentity());
configAction.setUserName(user.getUserName());
configAction.setOperation(Operation.Configure);
configAction.setTimestamp(new Date());
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ControllerServiceAuditor.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ControllerServiceAuditor.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ControllerServiceAuditor.java
index 0187ee4..af8428d 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ControllerServiceAuditor.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ControllerServiceAuditor.java
@@ -167,7 +167,7 @@ public class ControllerServiceAuditor extends NiFiAuditor {
// create a configuration action
FlowChangeAction configurationAction = new FlowChangeAction();
- configurationAction.setUserIdentity(user.getDn());
+ configurationAction.setUserIdentity(user.getIdentity());
configurationAction.setUserName(user.getUserName());
configurationAction.setOperation(operation);
configurationAction.setTimestamp(actionTimestamp);
@@ -187,7 +187,7 @@ public class ControllerServiceAuditor extends NiFiAuditor {
if (isDisabled != updateIsDisabled) {
// create a controller service action
FlowChangeAction serviceAction = new FlowChangeAction();
- serviceAction.setUserIdentity(user.getDn());
+ serviceAction.setUserIdentity(user.getIdentity());
serviceAction.setUserName(user.getUserName());
serviceAction.setTimestamp(new Date());
serviceAction.setSourceId(controllerService.getIdentifier());
@@ -271,7 +271,7 @@ public class ControllerServiceAuditor extends NiFiAuditor {
// create a processor action
FlowChangeAction processorAction = new FlowChangeAction();
- processorAction.setUserIdentity(user.getDn());
+ processorAction.setUserIdentity(user.getIdentity());
processorAction.setUserName(user.getUserName());
processorAction.setTimestamp(new Date());
processorAction.setSourceId(processor.getIdentifier());
@@ -289,7 +289,7 @@ public class ControllerServiceAuditor extends NiFiAuditor {
// create a reporting task action
FlowChangeAction reportingTaskAction = new FlowChangeAction();
- reportingTaskAction.setUserIdentity(user.getDn());
+ reportingTaskAction.setUserIdentity(user.getIdentity());
reportingTaskAction.setUserName(user.getUserName());
reportingTaskAction.setTimestamp(new Date());
reportingTaskAction.setSourceId(reportingTask.getIdentifier());
@@ -307,7 +307,7 @@ public class ControllerServiceAuditor extends NiFiAuditor {
// create a controller service action
FlowChangeAction serviceAction = new FlowChangeAction();
- serviceAction.setUserIdentity(user.getDn());
+ serviceAction.setUserIdentity(user.getIdentity());
serviceAction.setUserName(user.getUserName());
serviceAction.setTimestamp(new Date());
serviceAction.setSourceId(controllerService.getIdentifier());
@@ -387,7 +387,7 @@ public class ControllerServiceAuditor extends NiFiAuditor {
// create the controller service action for adding this controller service
action = new FlowChangeAction();
- action.setUserIdentity(user.getDn());
+ action.setUserIdentity(user.getIdentity());
action.setUserName(user.getUserName());
action.setOperation(operation);
action.setTimestamp(new Date());
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/FunnelAuditor.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/FunnelAuditor.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/FunnelAuditor.java
index 51cb20c..3949028 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/FunnelAuditor.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/FunnelAuditor.java
@@ -121,7 +121,7 @@ public class FunnelAuditor extends NiFiAuditor {
if (user != null) {
// create the action for adding this funnel
action = new FlowChangeAction();
- action.setUserIdentity(user.getDn());
+ action.setUserIdentity(user.getIdentity());
action.setUserName(user.getUserName());
action.setOperation(operation);
action.setTimestamp(new Date());
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/PortAuditor.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/PortAuditor.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/PortAuditor.java
index b07d64f..e99a1aa 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/PortAuditor.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/PortAuditor.java
@@ -205,7 +205,7 @@ public class PortAuditor extends NiFiAuditor {
for (ActionDetails detail : configurationDetails) {
// create the port action for updating the name
FlowChangeAction portAction = new FlowChangeAction();
- portAction.setUserIdentity(user.getDn());
+ portAction.setUserIdentity(user.getIdentity());
portAction.setUserName(user.getUserName());
portAction.setOperation(Operation.Configure);
portAction.setTimestamp(timestamp);
@@ -225,7 +225,7 @@ public class PortAuditor extends NiFiAuditor {
if (scheduledState != updatedScheduledState) {
// create a processor action
FlowChangeAction processorAction = new FlowChangeAction();
- processorAction.setUserIdentity(user.getDn());
+ processorAction.setUserIdentity(user.getIdentity());
processorAction.setUserName(user.getUserName());
processorAction.setTimestamp(new Date());
processorAction.setSourceId(updatedPort.getIdentifier());
@@ -323,7 +323,7 @@ public class PortAuditor extends NiFiAuditor {
// create the port action for adding this processor
action = new FlowChangeAction();
- action.setUserIdentity(user.getDn());
+ action.setUserIdentity(user.getIdentity());
action.setUserName(user.getUserName());
action.setOperation(operation);
action.setTimestamp(new Date());
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ProcessGroupAuditor.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ProcessGroupAuditor.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ProcessGroupAuditor.java
index d563555..89871e6 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ProcessGroupAuditor.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ProcessGroupAuditor.java
@@ -140,7 +140,7 @@ public class ProcessGroupAuditor extends NiFiAuditor {
// create the port action for updating the name
FlowChangeAction processGroupAction = new FlowChangeAction();
- processGroupAction.setUserIdentity(user.getDn());
+ processGroupAction.setUserIdentity(user.getIdentity());
processGroupAction.setUserName(user.getUserName());
processGroupAction.setOperation(operation);
processGroupAction.setTimestamp(timestamp);
@@ -157,7 +157,7 @@ public class ProcessGroupAuditor extends NiFiAuditor {
if (processGroupDTO.isRunning() != null) {
// create a process group action
FlowChangeAction processGroupAction = new FlowChangeAction();
- processGroupAction.setUserIdentity(user.getDn());
+ processGroupAction.setUserIdentity(user.getIdentity());
processGroupAction.setUserName(user.getUserName());
processGroupAction.setSourceId(processGroup.getIdentifier());
processGroupAction.setSourceName(processGroup.getName());
@@ -242,7 +242,7 @@ public class ProcessGroupAuditor extends NiFiAuditor {
// create the process group action for adding this process group
action = new FlowChangeAction();
- action.setUserIdentity(user.getDn());
+ action.setUserIdentity(user.getIdentity());
action.setUserName(user.getUserName());
action.setOperation(operation);
action.setTimestamp(new Date());
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ProcessorAuditor.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ProcessorAuditor.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ProcessorAuditor.java
index b8a2c69..4f147fb 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ProcessorAuditor.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ProcessorAuditor.java
@@ -177,7 +177,7 @@ public class ProcessorAuditor extends NiFiAuditor {
// create a configuration action
FlowChangeAction configurationAction = new FlowChangeAction();
- configurationAction.setUserIdentity(user.getDn());
+ configurationAction.setUserIdentity(user.getIdentity());
configurationAction.setUserName(user.getUserName());
configurationAction.setOperation(operation);
configurationAction.setTimestamp(actionTimestamp);
@@ -197,7 +197,7 @@ public class ProcessorAuditor extends NiFiAuditor {
if (scheduledState != updatedScheduledState) {
// create a processor action
FlowChangeAction processorAction = new FlowChangeAction();
- processorAction.setUserIdentity(user.getDn());
+ processorAction.setUserIdentity(user.getIdentity());
processorAction.setUserName(user.getUserName());
processorAction.setTimestamp(new Date());
processorAction.setSourceId(processor.getIdentifier());
@@ -294,7 +294,7 @@ public class ProcessorAuditor extends NiFiAuditor {
// create the processor action for adding this processor
action = new FlowChangeAction();
- action.setUserIdentity(user.getDn());
+ action.setUserIdentity(user.getIdentity());
action.setUserName(user.getUserName());
action.setOperation(operation);
action.setTimestamp(new Date());
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/RelationshipAuditor.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/RelationshipAuditor.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/RelationshipAuditor.java
index f90d572..95000d8 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/RelationshipAuditor.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/RelationshipAuditor.java
@@ -188,7 +188,7 @@ public class RelationshipAuditor extends NiFiAuditor {
// create a configuration action
FlowChangeAction configurationAction = new FlowChangeAction();
- configurationAction.setUserIdentity(user.getDn());
+ configurationAction.setUserIdentity(user.getIdentity());
configurationAction.setUserName(user.getUserName());
configurationAction.setOperation(Operation.Configure);
configurationAction.setTimestamp(actionTimestamp);
@@ -353,7 +353,7 @@ public class RelationshipAuditor extends NiFiAuditor {
// create a new relationship action
action = new FlowChangeAction();
- action.setUserIdentity(user.getDn());
+ action.setUserIdentity(user.getIdentity());
action.setUserName(user.getUserName());
action.setOperation(operation);
action.setTimestamp(actionTimestamp);
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/RemoteProcessGroupAuditor.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/RemoteProcessGroupAuditor.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/RemoteProcessGroupAuditor.java
index e145a62..5815634 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/RemoteProcessGroupAuditor.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/RemoteProcessGroupAuditor.java
@@ -246,7 +246,7 @@ public class RemoteProcessGroupAuditor extends NiFiAuditor {
for (ActionDetails detail : details) {
// create the port action for updating the name
FlowChangeAction remoteProcessGroupAction = new FlowChangeAction();
- remoteProcessGroupAction.setUserIdentity(user.getDn());
+ remoteProcessGroupAction.setUserIdentity(user.getIdentity());
remoteProcessGroupAction.setUserName(user.getUserName());
remoteProcessGroupAction.setOperation(Operation.Configure);
remoteProcessGroupAction.setTimestamp(timestamp);
@@ -267,7 +267,7 @@ public class RemoteProcessGroupAuditor extends NiFiAuditor {
if (transmissionState != updatedTransmissionState) {
// create a processor action
FlowChangeAction remoteProcessGroupAction = new FlowChangeAction();
- remoteProcessGroupAction.setUserIdentity(user.getDn());
+ remoteProcessGroupAction.setUserIdentity(user.getIdentity());
remoteProcessGroupAction.setUserName(user.getUserName());
remoteProcessGroupAction.setTimestamp(new Date());
remoteProcessGroupAction.setSourceId(updatedRemoteProcessGroup.getIdentifier());
@@ -356,7 +356,7 @@ public class RemoteProcessGroupAuditor extends NiFiAuditor {
// create the remote process group action
action = new FlowChangeAction();
- action.setUserIdentity(user.getDn());
+ action.setUserIdentity(user.getIdentity());
action.setUserName(user.getUserName());
action.setOperation(operation);
action.setTimestamp(new Date());
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ReportingTaskAuditor.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ReportingTaskAuditor.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ReportingTaskAuditor.java
index 712f99a..77df12a 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ReportingTaskAuditor.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ReportingTaskAuditor.java
@@ -160,7 +160,7 @@ public class ReportingTaskAuditor extends NiFiAuditor {
// create a configuration action
FlowChangeAction configurationAction = new FlowChangeAction();
- configurationAction.setUserIdentity(user.getDn());
+ configurationAction.setUserIdentity(user.getIdentity());
configurationAction.setUserName(user.getUserName());
configurationAction.setOperation(operation);
configurationAction.setTimestamp(actionTimestamp);
@@ -180,7 +180,7 @@ public class ReportingTaskAuditor extends NiFiAuditor {
if (scheduledState != updatedScheduledState) {
// create a reporting task action
FlowChangeAction taskAction = new FlowChangeAction();
- taskAction.setUserIdentity(user.getDn());
+ taskAction.setUserIdentity(user.getIdentity());
taskAction.setUserName(user.getUserName());
taskAction.setTimestamp(new Date());
taskAction.setSourceId(reportingTask.getIdentifier());
@@ -276,7 +276,7 @@ public class ReportingTaskAuditor extends NiFiAuditor {
// create the reporting task action for adding this reporting task
action = new FlowChangeAction();
- action.setUserIdentity(user.getDn());
+ action.setUserIdentity(user.getIdentity());
action.setUserName(user.getUserName());
action.setOperation(operation);
action.setTimestamp(new Date());
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/SnippetAuditor.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/SnippetAuditor.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/SnippetAuditor.java
index 34382b3..4b7c38a 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/SnippetAuditor.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/SnippetAuditor.java
@@ -232,7 +232,7 @@ public class SnippetAuditor extends NiFiAuditor {
if (user != null) {
// create the action for adding this funnel
action = new FlowChangeAction();
- action.setUserIdentity(user.getDn());
+ action.setUserIdentity(user.getIdentity());
action.setUserName(user.getUserName());
action.setOperation(operation);
action.setTimestamp(timestamp);
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java
index 2d3355a..73d76bd 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java
@@ -1267,6 +1267,13 @@ public interface NiFiServiceFacade {
Collection<UserDTO> getUsers(Boolean grouped);
/**
+ * Creates a new account request.
+ *
+ * @return user
+ */
+ UserDTO createUser();
+
+ /**
* Updates the specified user accordingly.
*
* @param user The user to update
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiWebApiConfiguration.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiWebApiConfiguration.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiWebApiConfiguration.java
new file mode 100644
index 0000000..58b0af8
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiWebApiConfiguration.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web;
+
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Import;
+import org.springframework.context.annotation.ImportResource;
+
+/**
+ *
+ */
+@Configuration
+@Import({NiFiWebApiSecurityConfiguration.class})
+@ImportResource({"classpath:nifi-context.xml",
+ "classpath:nifi-administration-context.xml",
+ "classpath:nifi-cluster-manager-context.xml",
+ "classpath:nifi-cluster-protocol-context.xml",
+ "classpath:nifi-web-security-context.xml",
+ "classpath:nifi-web-api-context.xml"})
+public class NiFiWebApiConfiguration {
+
+ public NiFiWebApiConfiguration() {
+ super();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiWebApiSecurityConfiguration.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiWebApiSecurityConfiguration.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiWebApiSecurityConfiguration.java
new file mode 100644
index 0000000..e8ed267
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiWebApiSecurityConfiguration.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web;
+
+import org.apache.nifi.admin.service.UserService;
+import org.apache.nifi.authentication.LoginIdentityProvider;
+import org.apache.nifi.util.NiFiProperties;
+import org.apache.nifi.web.security.NiFiAuthenticationProvider;
+import org.apache.nifi.web.security.anonymous.NiFiAnonymousUserFilter;
+import org.apache.nifi.web.security.NiFiAuthenticationEntryPoint;
+import org.apache.nifi.web.security.jwt.JwtAuthenticationFilter;
+import org.apache.nifi.web.security.jwt.JwtService;
+import org.apache.nifi.web.security.node.NodeAuthorizedUserFilter;
+import org.apache.nifi.web.security.token.NiFiAuthenticationRequestToken;
+import org.apache.nifi.web.security.x509.X509AuthenticationFilter;
+import org.apache.nifi.web.security.x509.X509CertificateExtractor;
+import org.apache.nifi.web.security.x509.X509IdentityProvider;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.security.authentication.AuthenticationManager;
+import org.springframework.security.config.annotation.authentication.builders.AuthenticationManagerBuilder;
+import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity;
+import org.springframework.security.config.annotation.web.builders.HttpSecurity;
+import org.springframework.security.config.annotation.web.builders.WebSecurity;
+import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
+import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter;
+import org.springframework.security.config.http.SessionCreationPolicy;
+import org.springframework.security.core.userdetails.AuthenticationUserDetailsService;
+import org.springframework.security.web.authentication.AnonymousAuthenticationFilter;
+
+/**
+ * NiFi Web Api Spring security
+ */
+@Configuration
+@EnableWebSecurity
+@EnableGlobalMethodSecurity(prePostEnabled = true)
+public class NiFiWebApiSecurityConfiguration extends WebSecurityConfigurerAdapter {
+
+ private NiFiProperties properties;
+ private UserService userService;
+ private AuthenticationUserDetailsService userDetailsService;
+ private JwtService jwtService;
+ private X509CertificateExtractor certificateExtractor;
+ private X509IdentityProvider certificateIdentityProvider;
+ private LoginIdentityProvider loginIdentityProvider;
+
+ public NiFiWebApiSecurityConfiguration() {
+ super(true); // disable defaults
+ }
+
+ @Override
+ public void configure(WebSecurity webSecurity) throws Exception {
+ webSecurity
+ .ignoring()
+ .antMatchers("/access/**");
+ }
+
+ @Override
+ protected void configure(HttpSecurity http) throws Exception {
+ http
+ .rememberMe().disable()
+ .exceptionHandling()
+ .authenticationEntryPoint(new NiFiAuthenticationEntryPoint(properties))
+ .and()
+ .authorizeRequests()
+ .anyRequest().fullyAuthenticated()
+ .and()
+ .sessionManagement()
+ .sessionCreationPolicy(SessionCreationPolicy.STATELESS);
+
+ // cluster authorized user
+ http.addFilterBefore(buildNodeAuthorizedUserFilter(), AnonymousAuthenticationFilter.class);
+
+ // anonymous
+ http.anonymous().authenticationFilter(buildAnonymousFilter());
+
+ // x509
+ http.addFilterAfter(buildX509Filter(), AnonymousAuthenticationFilter.class);
+
+ // jwt - consider when configured for log in
+ if (loginIdentityProvider != null) {
+ http.addFilterAfter(buildJwtFilter(), AnonymousAuthenticationFilter.class);
+ }
+ }
+
+ @Bean
+ @Override
+ public AuthenticationManager authenticationManagerBean() throws Exception {
+ // override xxxBean method so the authentication manager is available in app context (necessary for the method level security)
+ return super.authenticationManagerBean();
+ }
+
+ @Override
+ protected void configure(AuthenticationManagerBuilder auth) throws Exception {
+ auth.authenticationProvider(new NiFiAuthenticationProvider(userDetailsService));
+ }
+
+ private NodeAuthorizedUserFilter buildNodeAuthorizedUserFilter() {
+ final NodeAuthorizedUserFilter nodeFilter = new NodeAuthorizedUserFilter();
+ nodeFilter.setProperties(properties);
+ nodeFilter.setCertificateExtractor(certificateExtractor);
+ nodeFilter.setCertificateIdentityProvider(certificateIdentityProvider);
+ return nodeFilter;
+ }
+
+ private JwtAuthenticationFilter buildJwtFilter() throws Exception {
+ final JwtAuthenticationFilter jwtFilter = new JwtAuthenticationFilter();
+ jwtFilter.setProperties(properties);
+ jwtFilter.setJwtService(jwtService);
+ jwtFilter.setAuthenticationManager(authenticationManager());
+ return jwtFilter;
+ }
+
+ private X509AuthenticationFilter buildX509Filter() throws Exception {
+ final X509AuthenticationFilter x509Filter = new X509AuthenticationFilter();
+ x509Filter.setProperties(properties);
+ x509Filter.setCertificateExtractor(certificateExtractor);
+ x509Filter.setCertificateIdentityProvider(certificateIdentityProvider);
+ x509Filter.setAuthenticationManager(authenticationManager());
+ return x509Filter;
+ }
+
+ private AnonymousAuthenticationFilter buildAnonymousFilter() {
+ final NiFiAnonymousUserFilter anonymousFilter = new NiFiAnonymousUserFilter();
+ anonymousFilter.setUserService(userService);
+ return anonymousFilter;
+ }
+
+ @Autowired
+ public void setUserDetailsService(AuthenticationUserDetailsService<NiFiAuthenticationRequestToken> userDetailsService) {
+ this.userDetailsService = userDetailsService;
+ }
+
+ @Autowired
+ public void setUserService(UserService userService) {
+ this.userService = userService;
+ }
+
+ @Autowired
+ public void setProperties(NiFiProperties properties) {
+ this.properties = properties;
+ }
+
+ @Autowired
+ public void setJwtService(JwtService jwtService) {
+ this.jwtService = jwtService;
+ }
+
+ @Autowired
+ public void setLoginIdentityProvider(LoginIdentityProvider loginIdentityProvider) {
+ this.loginIdentityProvider = loginIdentityProvider;
+ }
+
+ @Autowired
+ public void setCertificateExtractor(X509CertificateExtractor certificateExtractor) {
+ this.certificateExtractor = certificateExtractor;
+ }
+
+ @Autowired
+ public void setCertificateIdentityProvider(X509IdentityProvider certificateIdentityProvider) {
+ this.certificateIdentityProvider = certificateIdentityProvider;
+ }
+
+}
[29/50] [abbrv] nifi git commit: NIFI-1086 Provide refactoring of
InvokeHTTP NIFI-980 Add support for HTTP Digest authentication to InvokeHttp
NIFI-1080 Provide additional InvokeHttp unit tests NIFI-1133 InvokeHTTP
Processor does not save Location header
Posted by mc...@apache.org.
NIFI-1086 Provide refactoring of InvokeHTTP
NIFI-980 Add support for HTTP Digest authentication to InvokeHttp
NIFI-1080 Provide additional InvokeHttp unit tests
NIFI-1133 InvokeHTTP Processor does not save Location header for 3xx responses
NIFI-1009 InvokeHTTP should be able to be scheduled without any incoming connection for GET operations
NIFI-61 Multiple improvements for InvokeHTTP inclusive of providing unique tx.id across clusters, dynamic HTTP header properties
Signed-off-by: Aldrin Piri <al...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/8c2323dc
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/8c2323dc
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/8c2323dc
Branch: refs/heads/NIFI-655
Commit: 8c2323dc8d0e107f1a99898370c7515fa9603122
Parents: fb335ea
Author: Joseph Percivall <jo...@yahoo.com>
Authored: Mon Nov 2 15:45:20 2015 -0500
Committer: Aldrin Piri <al...@apache.org>
Committed: Thu Nov 19 01:40:21 2015 -0500
----------------------------------------------------------------------
.../nifi-standard-processors/pom.xml | 52 +-
.../nifi/processors/standard/InvokeHTTP.java | 1119 ++++++++++--------
.../processors/standard/TestInvokeHTTP.java | 91 +-
.../processors/standard/TestInvokeHttpSSL.java | 3 +-
.../standard/util/TestInvokeHttpCommon.java | 1029 ++++++++++++----
pom.xml | 38 +
6 files changed, 1603 insertions(+), 729 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/8c2323dc/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
index 76f9daf..0427927 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
@@ -126,11 +126,6 @@ language governing permissions and limitations under the License. -->
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
- <artifactId>nifi-mock</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
<artifactId>nifi-socket-utils</artifactId>
</dependency>
<dependency>
@@ -138,11 +133,6 @@ language governing permissions and limitations under the License. -->
<artifactId>nifi-load-distribution-service-api</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.nifi</groupId>
- <artifactId>nifi-distributed-cache-client-service</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
</dependency>
@@ -154,21 +144,11 @@ language governing permissions and limitations under the License. -->
<groupId>org.apache.activemq</groupId>
<artifactId>activemq-client</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.activemq</groupId>
- <artifactId>activemq-broker</artifactId>
- <scope>test</scope>
- </dependency>
<dependency>
<groupId>com.jayway.jsonpath</groupId>
<artifactId>json-path</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.nifi</groupId>
- <artifactId>nifi-ssl-context-service</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-core</artifactId>
<version>1.7</version>
@@ -189,7 +169,37 @@ language governing permissions and limitations under the License. -->
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-asl</artifactId>
</dependency>
-
+ <dependency>
+ <groupId>com.squareup.okhttp</groupId>
+ <artifactId>okhttp</artifactId>
+ <version>2.5.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.burgstaller</groupId>
+ <artifactId>okhttp-digest</artifactId>
+ <version>0.4</version>
+ <type>jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-mock</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-distributed-cache-client-service</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.activemq</groupId>
+ <artifactId>activemq-broker</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-ssl-context-service</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.apache.derby</groupId>
<artifactId>derby</artifactId>
http://git-wip-us.apache.org/repos/asf/nifi/blob/8c2323dc/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/InvokeHTTP.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/InvokeHTTP.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/InvokeHTTP.java
index d827658..2a9760d 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/InvokeHTTP.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/InvokeHTTP.java
@@ -18,13 +18,8 @@ package org.apache.nifi.processors.standard;
import static org.apache.commons.lang3.StringUtils.trimToEmpty;
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.HttpURLConnection;
import java.net.InetSocketAddress;
import java.net.Proxy;
import java.net.Proxy.Type;
@@ -41,30 +36,49 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.net.ssl.HostnameVerifier;
-import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSession;
-import org.apache.commons.codec.binary.Base64;
+import com.burgstaller.okhttp.AuthenticationCacheInterceptor;
+import com.burgstaller.okhttp.CachingAuthenticatorDecorator;
+import com.burgstaller.okhttp.DispatchingAuthenticator;
+import com.burgstaller.okhttp.digest.CachingAuthenticator;
+import com.burgstaller.okhttp.digest.DigestAuthenticator;
+
+import com.squareup.okhttp.MediaType;
+import com.squareup.okhttp.OkHttpClient;
+import com.squareup.okhttp.Request;
+import com.squareup.okhttp.RequestBody;
+import com.squareup.okhttp.Response;
+
+import com.squareup.okhttp.ResponseBody;
+import okio.BufferedSink;
+import org.apache.commons.io.input.TeeInputStream;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.annotation.behavior.DynamicProperty;
import org.apache.nifi.annotation.behavior.InputRequirement;
import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.TriggerWhenEmpty;
import org.apache.nifi.annotation.behavior.WritesAttribute;
import org.apache.nifi.annotation.behavior.WritesAttributes;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.ValidationContext;
import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.expression.AttributeExpression;
import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
import org.apache.nifi.logging.ProcessorLog;
import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.ProcessContext;
@@ -72,118 +86,61 @@ import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.processors.standard.util.SoftLimitBoundedByteArrayOutputStream;
import org.apache.nifi.ssl.SSLContextService;
import org.apache.nifi.ssl.SSLContextService.ClientAuth;
+import org.apache.nifi.stream.io.StreamUtils;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
@SupportsBatching
@Tags({"http", "https", "rest", "client"})
-@InputRequirement(Requirement.INPUT_REQUIRED)
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@TriggerWhenEmpty
@CapabilityDescription("An HTTP client processor which converts FlowFile attributes to HTTP headers, with configurable HTTP method, url, etc.")
@WritesAttributes({
@WritesAttribute(attribute = "invokehttp.status.code", description = "The status code that is returned"),
@WritesAttribute(attribute = "invokehttp.status.message", description = "The status message that is returned"),
- @WritesAttribute(attribute = "invokehttp.response.body", description = "The response body"),
+ @WritesAttribute(attribute = "invokehttp.response.body", description = "In the instance where the status code received is not a success (2xx) "
+ + "then the response body will be put to the 'invokehttp.response.body' attribute of the request FlowFile."),
@WritesAttribute(attribute = "invokehttp.request.url", description = "The request URL"),
@WritesAttribute(attribute = "invokehttp.tx.id", description = "The transaction ID that is returned after reading the response"),
- @WritesAttribute(attribute = "invokehttp.remote.dn", description = "The DN of the remote server")})
-@DynamicProperty(name = "Trusted Hostname", value = "A hostname", description = "Bypass the normal truststore hostname verifier to allow the specified (single) remote hostname as trusted "
- + "Enabling this property has MITM security implications, use wisely. Only valid with SSL (HTTPS) connections.")
+ @WritesAttribute(attribute = "invokehttp.remote.dn", description = "The DN of the remote server"),
+ @WritesAttribute(attribute = "user-defined", description = "If the 'Put Response Body In Attribute' property is set then whatever it is set to "
+ + "will become the attribute key and the value would be the body of the HTTP response.")})
+@DynamicProperty(name = "Header Name", value = "Attribute Expression Language", supportsExpressionLanguage = true, description = "Send request header "
+ + "with a key matching the Dynamic Property Key and a value created by evaluating the Attribute Expression Language set in the value "
+ + "of the Dynamic Property.")
public final class InvokeHTTP extends AbstractProcessor {
- @Override
- protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
- return Config.PROPERTIES;
- }
-
- @Override
- protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(String propertyDescriptorName) {
- if (Config.PROP_TRUSTED_HOSTNAME.getName().equalsIgnoreCase(propertyDescriptorName)) {
- return Config.PROP_TRUSTED_HOSTNAME;
- }
- return super.getSupportedDynamicPropertyDescriptor(propertyDescriptorName);
- }
-
- @Override
- public Set<Relationship> getRelationships() {
- return Config.RELATIONSHIPS;
- }
-
- private volatile Pattern attributesToSend = null;
-
- @Override
- public void onPropertyModified(final PropertyDescriptor descriptor, final String oldValue, final String newValue) {
- final String trimmedValue = StringUtils.trimToEmpty(newValue);
-
- // compile the attributes-to-send filter pattern
- if (Config.PROP_ATTRIBUTES_TO_SEND.getName().equalsIgnoreCase(descriptor.getName())) {
- if (newValue.isEmpty()) {
- attributesToSend = null;
- } else {
- attributesToSend = Pattern.compile(trimmedValue);
- }
- }
- }
-
- @Override
- protected Collection<ValidationResult> customValidate(final ValidationContext validationContext) {
- final List<ValidationResult> results = new ArrayList<>(1);
- final boolean proxyHostSet = validationContext.getProperty(Config.PROP_PROXY_HOST).isSet();
- final boolean proxyPortSet = validationContext.getProperty(Config.PROP_PROXY_PORT).isSet();
-
- if ((proxyHostSet && !proxyPortSet) || (!proxyHostSet && proxyPortSet)) {
- results.add(new ValidationResult.Builder().subject("Proxy Host and Port").valid(false).explanation("If Proxy Host or Proxy Port is set, both must be set").build());
- }
-
- return results;
- }
-
- @Override
- public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
- final FlowFile flowFile = session.get();
- if (flowFile == null) {
- return;
- }
-
- final SSLContextService sslService = context.getProperty(Config.PROP_SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class);
- final SSLContext sslContext = sslService == null ? null : sslService.createSSLContext(ClientAuth.NONE);
-
- Transaction transaction = new Transaction(getLogger(), sslContext, attributesToSend, context, session, flowFile);
- transaction.process();
- }
-
- /**
- * Stores properties, relationships, configuration values, hard coded strings, magic numbers, etc.
- */
- public interface Config {
- // flowfile attribute keys returned after reading the response
- String STATUS_CODE = "invokehttp.status.code";
- String STATUS_MESSAGE = "invokehttp.status.message";
- String RESPONSE_BODY = "invokehttp.response.body";
- String REQUEST_URL = "invokehttp.request.url";
- String TRANSACTION_ID = "invokehttp.tx.id";
- String REMOTE_DN = "invokehttp.remote.dn";
-
- // Set of flowfile attributes which we generally always ignore during
- // processing, including when converting http headers, copying attributes, etc.
- // This set includes our strings defined above as well as some standard flowfile
- // attributes.
- public static final Set<String> IGNORED_ATTRIBUTES = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(
+ // flowfile attribute keys returned after reading the response
+ public final static String STATUS_CODE = "invokehttp.status.code";
+ public final static String STATUS_MESSAGE = "invokehttp.status.message";
+ public final static String RESPONSE_BODY = "invokehttp.response.body";
+ public final static String REQUEST_URL = "invokehttp.request.url";
+ public final static String TRANSACTION_ID = "invokehttp.tx.id";
+ public final static String REMOTE_DN = "invokehttp.remote.dn";
+
+ // Set of flowfile attributes which we generally always ignore during
+ // processing, including when converting http headers, copying attributes, etc.
+ // This set includes our strings defined above as well as some standard flowfile
+ // attributes.
+ public static final Set<String> IGNORED_ATTRIBUTES = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(
STATUS_CODE, STATUS_MESSAGE, RESPONSE_BODY, REQUEST_URL, TRANSACTION_ID, REMOTE_DN,
"uuid", "filename", "path")));
- // properties
- public static final PropertyDescriptor PROP_METHOD = new PropertyDescriptor.Builder()
+ // properties
+ public static final PropertyDescriptor PROP_METHOD = new PropertyDescriptor.Builder()
.name("HTTP Method")
- .description("HTTP request method (GET, POST, PUT, DELETE, HEAD, OPTIONS).")
+ .description("HTTP request method (GET, POST, PUT, DELETE, HEAD, OPTIONS). Arbitrary methods are also supported but will "
+ + "be sent without a message body.")
.required(true)
.defaultValue("GET")
.expressionLanguageSupported(true)
- .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .addValidator(StandardValidators.createAttributeExpressionLanguageValidator(AttributeExpression.ResultType.STRING))
.build();
- public static final PropertyDescriptor PROP_URL = new PropertyDescriptor.Builder()
+ public static final PropertyDescriptor PROP_URL = new PropertyDescriptor.Builder()
.name("Remote URL")
.description("Remote URL which will be connected to, including scheme, host, port, path.")
.required(true)
@@ -191,7 +148,7 @@ public final class InvokeHTTP extends AbstractProcessor {
.addValidator(StandardValidators.URL_VALIDATOR)
.build();
- public static final PropertyDescriptor PROP_CONNECT_TIMEOUT = new PropertyDescriptor.Builder()
+ public static final PropertyDescriptor PROP_CONNECT_TIMEOUT = new PropertyDescriptor.Builder()
.name("Connection Timeout")
.description("Max wait time for connection to remote service.")
.required(true)
@@ -199,7 +156,7 @@ public final class InvokeHTTP extends AbstractProcessor {
.addValidator(StandardValidators.TIME_PERIOD_VALIDATOR)
.build();
- public static final PropertyDescriptor PROP_READ_TIMEOUT = new PropertyDescriptor.Builder()
+ public static final PropertyDescriptor PROP_READ_TIMEOUT = new PropertyDescriptor.Builder()
.name("Read Timeout")
.description("Max wait time for response from remote service.")
.required(true)
@@ -207,7 +164,7 @@ public final class InvokeHTTP extends AbstractProcessor {
.addValidator(StandardValidators.TIME_PERIOD_VALIDATOR)
.build();
- public static final PropertyDescriptor PROP_DATE_HEADER = new PropertyDescriptor.Builder()
+ public static final PropertyDescriptor PROP_DATE_HEADER = new PropertyDescriptor.Builder()
.name("Include Date Header")
.description("Include an RFC-2616 Date header in the request.")
.required(true)
@@ -216,7 +173,7 @@ public final class InvokeHTTP extends AbstractProcessor {
.addValidator(StandardValidators.BOOLEAN_VALIDATOR)
.build();
- public static final PropertyDescriptor PROP_FOLLOW_REDIRECTS = new PropertyDescriptor.Builder()
+ public static final PropertyDescriptor PROP_FOLLOW_REDIRECTS = new PropertyDescriptor.Builder()
.name("Follow Redirects")
.description("Follow HTTP redirects issued by remote server.")
.required(true)
@@ -225,48 +182,50 @@ public final class InvokeHTTP extends AbstractProcessor {
.addValidator(StandardValidators.BOOLEAN_VALIDATOR)
.build();
- public static final PropertyDescriptor PROP_ATTRIBUTES_TO_SEND = new PropertyDescriptor.Builder()
+ public static final PropertyDescriptor PROP_ATTRIBUTES_TO_SEND = new PropertyDescriptor.Builder()
.name("Attributes to Send")
.description("Regular expression that defines which attributes to send as HTTP headers in the request. "
- + "If not defined, no attributes are sent as headers.")
+ + "If not defined, no attributes are sent as headers. Also any dynamic properties set will be sent as headers. "
+ + "The dynamic property key will be the header key and the dynamic property value will be interpreted as expression "
+ + "language will be the header value.")
.required(false)
.addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR)
.build();
- public static final PropertyDescriptor PROP_SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder()
+ public static final PropertyDescriptor PROP_SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder()
.name("SSL Context Service")
.description("The SSL Context Service used to provide client certificate information for TLS/SSL (https) connections.")
.required(false)
.identifiesControllerService(SSLContextService.class)
.build();
- public static final PropertyDescriptor PROP_PROXY_HOST = new PropertyDescriptor.Builder()
+ public static final PropertyDescriptor PROP_PROXY_HOST = new PropertyDescriptor.Builder()
.name("Proxy Host")
.description("The fully qualified hostname or IP address of the proxy server")
.required(false)
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.build();
- public static final PropertyDescriptor PROP_PROXY_PORT = new PropertyDescriptor.Builder()
+ public static final PropertyDescriptor PROP_PROXY_PORT = new PropertyDescriptor.Builder()
.name("Proxy Port")
.description("The port of the proxy server")
.required(false)
.addValidator(StandardValidators.PORT_VALIDATOR)
.build();
- // Per RFC 7235, 2617, and 2616.
- // basic-credentials = base64-user-pass
- // base64-user-pass = userid ":" password
- // userid = *<TEXT excluding ":">
- // password = *TEXT
- //
- // OCTET = <any 8-bit sequence of data>
- // CTL = <any US-ASCII control character (octets 0 - 31) and DEL (127)>
- // LWS = [CRLF] 1*( SP | HT )
- // TEXT = <any OCTET except CTLs but including LWS>
- //
- // Per RFC 7230, username & password in URL are now disallowed in HTTP and HTTPS URIs.
- public static final PropertyDescriptor PROP_BASIC_AUTH_USERNAME = new PropertyDescriptor.Builder()
+ // Per RFC 7235, 2617, and 2616.
+ // basic-credentials = base64-user-pass
+ // base64-user-pass = userid ":" password
+ // userid = *<TEXT excluding ":">
+ // password = *TEXT
+ //
+ // OCTET = <any 8-bit sequence of data>
+ // CTL = <any US-ASCII control character (octets 0 - 31) and DEL (127)>
+ // LWS = [CRLF] 1*( SP | HT )
+ // TEXT = <any OCTET except CTLs but including LWS>
+ //
+ // Per RFC 7230, username & password in URL are now disallowed in HTTP and HTTPS URIs.
+ public static final PropertyDescriptor PROP_BASIC_AUTH_USERNAME = new PropertyDescriptor.Builder()
.name("Basic Authentication Username")
.displayName("Basic Authentication Username")
.description("The username to be used by the client to authenticate against the Remote URL. Cannot include control characters (0-31), ':', or DEL (127).")
@@ -274,7 +233,7 @@ public final class InvokeHTTP extends AbstractProcessor {
.addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
.build();
- public static final PropertyDescriptor PROP_BASIC_AUTH_PASSWORD = new PropertyDescriptor.Builder()
+ public static final PropertyDescriptor PROP_BASIC_AUTH_PASSWORD = new PropertyDescriptor.Builder()
.name("Basic Authentication Password")
.displayName("Basic Authentication Password")
.description("The password to be used by the client to authenticate against the Remote URL.")
@@ -283,7 +242,64 @@ public final class InvokeHTTP extends AbstractProcessor {
.addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
.build();
- public static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(
+ public static final PropertyDescriptor PROP_PUT_OUTPUT_IN_ATTRIBUTE = new PropertyDescriptor.Builder()
+ .name("Put Response Body In Attribute")
+ .description("If set, the response body received back will be put into an attribute of the original FlowFile instead of a separate "
+ + "FlowFile. The attribute key to put to is determined by evaluating value of this property. ")
+ .addValidator(StandardValidators.createAttributeExpressionLanguageValidator(AttributeExpression.ResultType.STRING))
+ .expressionLanguageSupported(true)
+ .build();
+
+ public static final PropertyDescriptor PROP_PUT_ATTRIBUTE_MAX_LENGTH = new PropertyDescriptor.Builder()
+ .name("Max Length To Put In Attribute")
+ .description("If routing the response body to an attribute of the original (by setting the \"Put response body in attribute\" "
+ + "property or by receiving an error status code), the number of characters put to the attribute value will be at "
+ + "most this amount. This is important because attributes are held in memory and large attributes will quickly "
+ + "cause out of memory issues. If the output goes longer than this value, it will be truncated to fit. "
+ + "Consider making this smaller if able.")
+ .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR)
+ .defaultValue("256")
+ .build();
+
+ public static final PropertyDescriptor PROP_DIGEST_AUTH = new PropertyDescriptor.Builder()
+ .name("Digest Authentication")
+ .displayName("Use Digest Authentication")
+ .description("Whether to communicate with the website using Digest Authentication. 'Basic Authentication Username' and 'Basic Authentication Password' are used "
+ + "for authentication.")
+ .required(false)
+ .defaultValue("false")
+ .allowableValues("true", "false")
+ .build();
+
+ public static final PropertyDescriptor PROP_OUTPUT_RESPONSE_REGARDLESS = new PropertyDescriptor.Builder()
+ .name("Always Output Response")
+ .description("Will force a response FlowFile to be generated and routed to the 'Response' relationship regardless of what the server status code received is "
+ + "or if the processor is configured to put the server response body in the request attribute. In the later configuration a request FlowFile with the "
+ + "response body in the attribute and a typical response FlowFile will be emitted to their respective relationships.")
+ .required(false)
+ .defaultValue("false")
+ .allowableValues("true", "false")
+ .build();
+
+ public static final PropertyDescriptor PROP_TRUSTED_HOSTNAME = new PropertyDescriptor.Builder()
+ .name("Trusted Hostname")
+ .description("Bypass the normal truststore hostname verifier to allow the specified remote hostname as trusted. "
+ + "Enabling this property has MITM security implications, use wisely. Will still accept other connections based "
+ + "on the normal truststore hostname verifier. Only valid with SSL (HTTPS) connections.")
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .required(false)
+ .build();
+
+ public static final PropertyDescriptor PROP_ADD_HEADERS_TO_REQUEST = new PropertyDescriptor.Builder()
+ .name("Add Response Headers to Request")
+ .description("Enabling this property saves all the response headers to the original request. This may be when the response headers are needed "
+ + "but a response is not generated due to the status code received.")
+ .required(false)
+ .defaultValue("false")
+ .allowableValues("true", "false")
+ .build();
+
+ public static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(
PROP_METHOD,
PROP_URL,
PROP_SSL_CONTEXT_SERVICE,
@@ -295,499 +311,626 @@ public final class InvokeHTTP extends AbstractProcessor {
PROP_BASIC_AUTH_USERNAME,
PROP_BASIC_AUTH_PASSWORD,
PROP_PROXY_HOST,
- PROP_PROXY_PORT));
-
- // property to allow the hostname verifier to be overridden
- // this is a "hidden" property - it's configured using a dynamic user property
- public static final PropertyDescriptor PROP_TRUSTED_HOSTNAME = new PropertyDescriptor.Builder()
- .name("Trusted Hostname")
- .description("Bypass the normal truststore hostname verifier to allow the specified remote hostname as trusted. "
- + "Enabling this property has MITM security implications, use wisely. Will still accept other connections based "
- + "on the normal truststore hostname verifier. Only valid with SSL (HTTPS) connections.")
- .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
- .dynamic(true)
- .build();
-
- // relationships
- public static final Relationship REL_SUCCESS_REQ = new Relationship.Builder()
+ PROP_PROXY_PORT,
+ PROP_PUT_OUTPUT_IN_ATTRIBUTE,
+ PROP_PUT_ATTRIBUTE_MAX_LENGTH,
+ PROP_DIGEST_AUTH,
+ PROP_OUTPUT_RESPONSE_REGARDLESS,
+ PROP_TRUSTED_HOSTNAME,
+ PROP_ADD_HEADERS_TO_REQUEST));
+
+ // relationships
+ public static final Relationship REL_SUCCESS_REQ = new Relationship.Builder()
.name("Original")
- .description("Original FlowFile will be routed upon success (2xx status codes).")
+ .description("The original FlowFile will be routed upon success (2xx status codes). It will have new attributes detailing the "
+ + "success of the request.")
.build();
- public static final Relationship REL_SUCCESS_RESP = new Relationship.Builder()
+ public static final Relationship REL_RESPONSE = new Relationship.Builder()
.name("Response")
- .description("Response FlowFile will be routed upon success (2xx status codes).")
+ .description("A Response FlowFile will be routed upon success (2xx status codes). If the 'Output Response Regardless' property "
+ + "is true then the response will be sent to this relationship regardless of the status code received.")
.build();
- public static final Relationship REL_RETRY = new Relationship.Builder()
+ public static final Relationship REL_RETRY = new Relationship.Builder()
.name("Retry")
- .description("FlowFile will be routed on any status code that can be retried (5xx status codes).")
+ .description("The original FlowFile will be routed on any status code that can be retried (5xx status codes). It will have new "
+ + "attributes detailing the request.")
.build();
- public static final Relationship REL_NO_RETRY = new Relationship.Builder()
+ public static final Relationship REL_NO_RETRY = new Relationship.Builder()
.name("No Retry")
- .description("FlowFile will be routed on any status code that should NOT be retried (1xx, 3xx, 4xx status codes).")
+ .description("The original FlowFile will be routed on any status code that should NOT be retried (1xx, 3xx, 4xx status codes). "
+ + "It will have new attributes detailing the request.")
.build();
- public static final Relationship REL_FAILURE = new Relationship.Builder()
+ public static final Relationship REL_FAILURE = new Relationship.Builder()
.name("Failure")
- .description("FlowFile will be routed on any type of connection failure, timeout or general exception.")
+ .description("The original FlowFile will be routed on any type of connection failure, timeout or general exception. "
+ + "It will have new attributes detailing the request.")
.build();
- public static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(
- REL_SUCCESS_REQ, REL_SUCCESS_RESP, REL_RETRY, REL_NO_RETRY, REL_FAILURE)));
+ public static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(
+ REL_SUCCESS_REQ, REL_RESPONSE, REL_RETRY, REL_NO_RETRY, REL_FAILURE)));
- }
+ private volatile Set<String> dynamicPropertyNames = new HashSet<>();
/**
- * A single invocation of an HTTP request/response from the InvokeHTTP processor. This class encapsulates the entirety of the flowfile processing.
- * <p>
- * This class is not thread safe and is created new for every flowfile processed.
+ * Pattern used to compute RFC 2616 Dates (#sec3.3.1). This format is used by the HTTP Date header and is optionally sent by the processor. This date is effectively an RFC 822/1123 date
+ * string, but HTTP requires it to be in GMT (preferring the literal 'GMT' string).
*/
- private static class Transaction implements Config {
-
- /**
- * Pattern used to compute RFC 2616 Dates (#sec3.3.1). This format is used by the HTTP Date header and is optionally sent by the processor. This date is effectively an RFC 822/1123 date
- * string, but HTTP requires it to be in GMT (preferring the literal 'GMT' string).
- */
- private static final String rfc1123 = "EEE, dd MMM yyyy HH:mm:ss 'GMT'";
- private static final DateTimeFormatter dateFormat = DateTimeFormat.forPattern(rfc1123).withLocale(Locale.US).withZoneUTC();
-
- /**
- * Every request/response cycle from this client has a unique transaction id which will be stored as a flowfile attribute. This generator is used to create the id.
- */
- private static final AtomicLong txIdGenerator = new AtomicLong();
-
- private static final Charset utf8 = Charset.forName("UTF-8");
-
- private final ProcessorLog logger;
- private final SSLContext sslContext;
- private final Pattern attributesToSend;
- private final ProcessContext context;
- private final ProcessSession session;
-
- private final long txId = txIdGenerator.incrementAndGet();
- private final long startNanos = System.nanoTime();
-
- private FlowFile request;
- private FlowFile response;
- private HttpURLConnection conn;
-
- private int statusCode;
- private String statusMessage;
-
- public Transaction(
- final ProcessorLog logger,
- final SSLContext sslContext,
- final Pattern attributesToSend,
- final ProcessContext context,
- final ProcessSession session,
- final FlowFile request) {
-
- this.logger = logger;
- this.sslContext = sslContext;
- this.attributesToSend = attributesToSend;
- this.context = context;
- this.session = session;
- this.request = request;
- }
+ private static final String RFC_1123 = "EEE, dd MMM yyyy HH:mm:ss 'GMT'";
+ private static final DateTimeFormatter DATE_FORMAT = DateTimeFormat.forPattern(RFC_1123).withLocale(Locale.US).withZoneUTC();
+ private final AtomicReference<OkHttpClient> okHttpClientAtomicReference = new AtomicReference<>();
- public void process() {
- try {
- openConnection();
- sendRequest();
- readResponse();
- transfer();
- } catch (final Exception e) {
- // log exception
- logger.error("Routing to {} due to exception: {}", new Object[] {REL_FAILURE.getName(), e}, e);
-
- // penalize
- request = session.penalize(request);
+ public static final String DEFAULT_CONTENT_TYPE = "application/octet-stream";
- // transfer original to failure
- session.transfer(request, REL_FAILURE);
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return PROPERTIES;
+ }
- // cleanup response flowfile, if applicable
- try {
- if (response != null) {
- session.remove(response);
- }
- } catch (final Exception e1) {
- logger.error("Could not cleanup response flowfile due to exception: {}", new Object[] {e1}, e1);
+ @Override
+ protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(String propertyDescriptorName) {
+ return new PropertyDescriptor.Builder()
+ .required(false)
+ .name(propertyDescriptorName)
+ .addValidator(StandardValidators.createAttributeExpressionLanguageValidator(AttributeExpression.ResultType.STRING, true))
+ .dynamic(true)
+ .expressionLanguageSupported(true)
+ .build();
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return RELATIONSHIPS;
+ }
+
+ private volatile Pattern regexAttributesToSend = null;
+
+ @Override
+ public void onPropertyModified(final PropertyDescriptor descriptor, final String oldValue, final String newValue) {
+ if (descriptor.isDynamic()) {
+ final Set<String> newDynamicPropertyNames = new HashSet<>(dynamicPropertyNames);
+ if (newValue == null) {
+ newDynamicPropertyNames.remove(descriptor.getName());
+ } else if (oldValue == null) { // new property
+ newDynamicPropertyNames.add(descriptor.getName());
+ }
+ this.dynamicPropertyNames = Collections.unmodifiableSet(newDynamicPropertyNames);
+ } else {
+ // compile the attributes-to-send filter pattern
+ if (PROP_ATTRIBUTES_TO_SEND.getName().equalsIgnoreCase(descriptor.getName())) {
+ if (newValue.isEmpty()) {
+ regexAttributesToSend = null;
+ } else {
+ final String trimmedValue = StringUtils.trimToEmpty(newValue);
+ regexAttributesToSend = Pattern.compile(trimmedValue);
}
}
}
+ }
- private void openConnection() throws IOException {
- // read the url property from the context
- final String urlstr = trimToEmpty(context.getProperty(PROP_URL).evaluateAttributeExpressions(request).getValue());
- final URL url = new URL(urlstr);
- final String authuser = trimToEmpty(context.getProperty(PROP_BASIC_AUTH_USERNAME).getValue());
- final String authpass = trimToEmpty(context.getProperty(PROP_BASIC_AUTH_PASSWORD).getValue());
-
- String authstrencoded = null;
- if (!authuser.isEmpty()) {
- String authstr = authuser + ":" + authpass;
- byte[] bytestrencoded = Base64.encodeBase64(authstr.getBytes(StandardCharsets.UTF_8));
- authstrencoded = new String(bytestrencoded, StandardCharsets.UTF_8);
- }
+ @Override
+ protected Collection<ValidationResult> customValidate(final ValidationContext validationContext) {
+ final List<ValidationResult> results = new ArrayList<>(1);
+ final boolean proxyHostSet = validationContext.getProperty(PROP_PROXY_HOST).isSet();
+ final boolean proxyPortSet = validationContext.getProperty(PROP_PROXY_PORT).isSet();
- // create the connection
- final String proxyHost = context.getProperty(PROP_PROXY_HOST).getValue();
- final Integer proxyPort = context.getProperty(PROP_PROXY_PORT).asInteger();
- if (proxyHost == null || proxyPort == null) {
- conn = (HttpURLConnection) url.openConnection();
- } else {
- final Proxy proxy = new Proxy(Type.HTTP, new InetSocketAddress(proxyHost, proxyPort));
- conn = (HttpURLConnection) url.openConnection(proxy);
- }
+ if ((proxyHostSet && !proxyPortSet) || (!proxyHostSet && proxyPortSet)) {
+ results.add(new ValidationResult.Builder().subject("Proxy Host and Port").valid(false).explanation("If Proxy Host or Proxy Port is set, both must be set").build());
+ }
- if (authstrencoded != null) {
- conn.setRequestProperty("Authorization", "Basic " + authstrencoded);
- }
+ return results;
+ }
- // set the request method
- String method = trimToEmpty(context.getProperty(PROP_METHOD).evaluateAttributeExpressions(request).getValue()).toUpperCase();
- conn.setRequestMethod(method);
+ @OnScheduled
+ public void setUpClient(final ProcessContext context) throws IOException {
+ okHttpClientAtomicReference.set(null);
- // set timeouts
- conn.setConnectTimeout(context.getProperty(PROP_CONNECT_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
- conn.setReadTimeout(context.getProperty(PROP_READ_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
+ OkHttpClient okHttpClient = new OkHttpClient();
- // set whether to follow redirects
- conn.setInstanceFollowRedirects(context.getProperty(PROP_FOLLOW_REDIRECTS).asBoolean());
+ // Add a proxy if set
+ final String proxyHost = context.getProperty(PROP_PROXY_HOST).getValue();
+ final Integer proxyPort = context.getProperty(PROP_PROXY_PORT).asInteger();
+ if (proxyHost != null && proxyPort != null) {
+ final Proxy proxy = new Proxy(Type.HTTP, new InetSocketAddress(proxyHost, proxyPort));
+ okHttpClient.setProxy(proxy);
+ }
- // special handling for https
- if (conn instanceof HttpsURLConnection) {
- HttpsURLConnection sconn = (HttpsURLConnection) conn;
+ // Set timeouts
+ okHttpClient.setConnectTimeout((context.getProperty(PROP_CONNECT_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue()), TimeUnit.MILLISECONDS);
+ okHttpClient.setReadTimeout(context.getProperty(PROP_READ_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue(), TimeUnit.MILLISECONDS);
- // check if the ssl context is set
- if (sslContext != null) {
- sconn.setSSLSocketFactory(sslContext.getSocketFactory());
- }
+ // Set whether to follow redirects
+ okHttpClient.setFollowRedirects(context.getProperty(PROP_FOLLOW_REDIRECTS).asBoolean());
- // check the trusted hostname property and override the HostnameVerifier
- String trustedHostname = trimToEmpty(context.getProperty(PROP_TRUSTED_HOSTNAME).getValue());
- if (!trustedHostname.isEmpty()) {
- sconn.setHostnameVerifier(new OverrideHostnameVerifier(trustedHostname, sconn.getHostnameVerifier()));
- }
- }
+ final SSLContextService sslService = context.getProperty(PROP_SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class);
+ final SSLContext sslContext = sslService == null ? null : sslService.createSSLContext(ClientAuth.NONE);
+ // check if the ssl context is set and add the factory if so
+ if (sslContext != null) {
+ okHttpClient.setSslSocketFactory(sslContext.getSocketFactory());
}
- private void sendRequest() throws IOException {
- // set the http request properties using flowfile attribute values
- setRequestProperties();
+ // check the trusted hostname property and override the HostnameVerifier
+ String trustedHostname = trimToEmpty(context.getProperty(PROP_TRUSTED_HOSTNAME).getValue());
+ if (!trustedHostname.isEmpty()) {
+ okHttpClient.setHostnameVerifier(new OverrideHostnameVerifier(trustedHostname, okHttpClient.getHostnameVerifier()));
+ }
- // log request
- logRequest();
+ final String authUser = trimToEmpty(context.getProperty(PROP_BASIC_AUTH_USERNAME).getValue());
- // we only stream data for POST and PUT requests
- String method = conn.getRequestMethod().toUpperCase();
- if ("POST".equals(method) || "PUT".equals(method)) {
- conn.setDoOutput(true);
- conn.setFixedLengthStreamingMode(request.getSize());
+ // If the username/password properties are set then check if digest auth is being used
+ if (!authUser.isEmpty() && "true".equalsIgnoreCase(context.getProperty(PROP_DIGEST_AUTH).getValue())) {
+ final String authPass = trimToEmpty(context.getProperty(PROP_BASIC_AUTH_PASSWORD).getValue());
- // write the flowfile contents to the output stream
- try (OutputStream os = new BufferedOutputStream(conn.getOutputStream())) {
- session.exportTo(request, os);
- }
+ /*
+ * Currently OkHttp doesn't have built-in Digest Auth Support. The ticket for adding it is here:
+ * https://github.com/square/okhttp/issues/205#issuecomment-154047052
+ * Once added this should be refactored to use the built in support. For now, a third party lib is needed.
+ */
+ final Map<String, CachingAuthenticator> authCache = new ConcurrentHashMap<>();
+
+ com.burgstaller.okhttp.digest.Credentials credentials = new com.burgstaller.okhttp.digest.Credentials(authUser, authPass);
+
+ final DigestAuthenticator digestAuthenticator = new DigestAuthenticator(credentials);
+
+ DispatchingAuthenticator authenticator = new DispatchingAuthenticator.Builder()
+ .with("Digest", digestAuthenticator)
+ .build();
+
+ okHttpClient.interceptors().add(new AuthenticationCacheInterceptor(authCache));
+ okHttpClient.setAuthenticator(new CachingAuthenticatorDecorator(authenticator, authCache));
+ }
- // emit provenance event
- session.getProvenanceReporter().send(request, conn.getURL().toExternalForm());
+ okHttpClientAtomicReference.set(okHttpClient);
+ }
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ OkHttpClient okHttpClient = okHttpClientAtomicReference.get();
+
+ FlowFile requestFlowFile = session.get();
+
+ // Checking to see if the property to put the body of the response in an attribute was set
+ boolean putToAttribute = context.getProperty(PROP_PUT_OUTPUT_IN_ATTRIBUTE).isSet();
+ if (requestFlowFile == null) {
+ if(context.hasNonLoopConnection()){
+ return;
}
+ String request = context.getProperty(PROP_METHOD).evaluateAttributeExpressions().getValue().toUpperCase();
+ if ("POST".equals(request) || "PUT".equals(request)) {
+ return;
+ } else if (putToAttribute) {
+ requestFlowFile = session.create();
+ }
}
- private void readResponse() throws IOException {
+ // Setting some initial variables
+ final int maxAttributeSize = context.getProperty(PROP_PUT_ATTRIBUTE_MAX_LENGTH).asInteger();
+ final ProcessorLog logger = getLogger();
+
+ // Every request/response cycle has a unique transaction id which will be stored as a flowfile attribute.
+ final UUID txId = UUID.randomUUID();
+
+ FlowFile responseFlowFile = null;
+ try {
+ // read the url property from the context
+ final String urlstr = trimToEmpty(context.getProperty(PROP_URL).evaluateAttributeExpressions(requestFlowFile).getValue());
+ final URL url = new URL(urlstr);
+
+ Request httpRequest = configureRequest(context, session, requestFlowFile, url);
+
+ // log request
+ logRequest(logger, httpRequest);
+
+ // emit send provenance event if successfully sent to the server
+ if (httpRequest.body() != null) {
+ session.getProvenanceReporter().send(requestFlowFile, url.toExternalForm(), true);
+ }
+
+ final long startNanos = System.nanoTime();
+ Response responseHttp = okHttpClient.newCall(httpRequest).execute();
// output the raw response headers (DEBUG level only)
- logResponse();
+ logResponse(logger, url, responseHttp);
// store the status code and message
- statusCode = conn.getResponseCode();
- statusMessage = conn.getResponseMessage();
+ int statusCode = responseHttp.code();
+ String statusMessage = responseHttp.message();
+
+ if (statusCode == 0) {
+ throw new IllegalStateException("Status code unknown, connection hasn't been attempted.");
+ }
+
+ // Create a map of the status attributes that are always written to the request and reponse FlowFiles
+ Map<String, String> statusAttributes = new HashMap<>();
+ statusAttributes.put(STATUS_CODE, String.valueOf(statusCode));
+ statusAttributes.put(STATUS_MESSAGE, statusMessage);
+ statusAttributes.put(REQUEST_URL, url.toExternalForm());
+ statusAttributes.put(TRANSACTION_ID, txId.toString());
- // always write the status attributes to the request flowfile
- request = writeStatusAttributes(request);
+ if (requestFlowFile != null) {
+ requestFlowFile = session.putAllAttributes(requestFlowFile, statusAttributes);
+ }
- // read from the appropriate input stream
- try (InputStream is = getResponseStream()) {
+ // If the property to add the response headers to the request flowfile is true then add them
+ if (context.getProperty(PROP_ADD_HEADERS_TO_REQUEST).asBoolean() && requestFlowFile != null) {
+ // write the response headers as attributes
+ // this will overwrite any existing flowfile attributes
+ requestFlowFile = session.putAllAttributes(requestFlowFile, convertAttributesFromHeaders(url, responseHttp));
+ }
- // if not successful, store the response body into a flowfile attribute
- if (!isSuccess()) {
- String body = trimToEmpty(toString(is, utf8));
- request = session.putAttribute(request, RESPONSE_BODY, body);
+ boolean outputBodyToRequestAttribute = (!isSuccess(statusCode) || putToAttribute) && requestFlowFile != null;
+ boolean outputBodyToResponseContent = (isSuccess(statusCode) && !putToAttribute) || context.getProperty(PROP_OUTPUT_RESPONSE_REGARDLESS).asBoolean();
+ ResponseBody responseBody = responseHttp.body();
+ boolean bodyExists = responseBody != null;
+
+ InputStream responseBodyStream = null;
+ SoftLimitBoundedByteArrayOutputStream outputStreamToRequestAttribute = null;
+ TeeInputStream teeInputStream = null;
+ try {
+ responseBodyStream = bodyExists ? responseBody.byteStream() : null;
+ if (responseBodyStream != null && outputBodyToRequestAttribute && outputBodyToResponseContent) {
+ outputStreamToRequestAttribute = new SoftLimitBoundedByteArrayOutputStream(maxAttributeSize);
+ teeInputStream = new TeeInputStream(responseBodyStream, outputStreamToRequestAttribute);
}
- // if successful, store the response body as the flowfile payload
- // we include additional flowfile attributes including the reponse headers
- // and the status codes.
- if (isSuccess()) {
+ if (outputBodyToResponseContent) {
+ /*
+ * If successful and putting to response flowfile, store the response body as the flowfile payload
+ * we include additional flowfile attributes including the response headers and the status codes.
+ */
+
// clone the flowfile to capture the response
- response = session.create(request);
+ if (requestFlowFile != null) {
+ responseFlowFile = session.create(requestFlowFile);
+ } else {
+ responseFlowFile = session.create();
+ }
// write the status attributes
- response = writeStatusAttributes(response);
+ responseFlowFile = session.putAllAttributes(responseFlowFile, statusAttributes);
// write the response headers as attributes
// this will overwrite any existing flowfile attributes
- response = session.putAllAttributes(response, convertAttributesFromHeaders());
+ responseFlowFile = session.putAllAttributes(responseFlowFile, convertAttributesFromHeaders(url, responseHttp));
// transfer the message body to the payload
// can potentially be null in edge cases
- if (is != null) {
- response = session.importFrom(is, response);
+ if (bodyExists) {
+ if (teeInputStream != null) {
+ responseFlowFile = session.importFrom(teeInputStream, responseFlowFile);
+ } else {
+ responseFlowFile = session.importFrom(responseBodyStream, responseFlowFile);
+ }
// emit provenance event
final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
- session.getProvenanceReporter().fetch(response, conn.getURL().toExternalForm(), millis);
+ session.getProvenanceReporter().fetch(responseFlowFile, url.toExternalForm(), millis);
+ }
+ }
+
+ // if not successful and request flowfile is not null, store the response body into a flowfile attribute
+ if (outputBodyToRequestAttribute && bodyExists) {
+ String attributeKey = context.getProperty(PROP_PUT_OUTPUT_IN_ATTRIBUTE).evaluateAttributeExpressions(requestFlowFile).getValue();
+ if (attributeKey == null) {
+ attributeKey = RESPONSE_BODY;
}
+ byte[] outputBuffer;
+ int size;
+
+ if (outputStreamToRequestAttribute != null) {
+ outputBuffer = outputStreamToRequestAttribute.getBuffer();
+ size = outputStreamToRequestAttribute.size();
+ } else {
+ outputBuffer = new byte[maxAttributeSize];
+ size = StreamUtils.fillBuffer(responseBodyStream, outputBuffer, false);
+ }
+ String bodyString = new String(outputBuffer, 0, size, getCharsetFromMediaType(responseBody.contentType()));
+ requestFlowFile = session.putAttribute(requestFlowFile, attributeKey, bodyString);
+ final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
+ session.getProvenanceReporter().modifyAttributes(requestFlowFile, "The " + attributeKey + " has been added. The value of which is the body of a http call to "
+ + url.toExternalForm() + ". It took " + millis + "millis,");
+ }
+ } finally {
+ if(outputStreamToRequestAttribute != null){
+ outputStreamToRequestAttribute.close();
+ outputStreamToRequestAttribute = null;
}
+ if(teeInputStream != null){
+ teeInputStream.close();
+ teeInputStream = null;
+ } else if(responseBodyStream != null){
+ responseBodyStream.close();
+ responseBodyStream = null;
+ }
+ }
+ route(requestFlowFile, responseFlowFile, session, context, statusCode);
+ } catch (final Exception e) {
+ // penalize or yield
+ if (requestFlowFile != null) {
+ logger.error("Routing to {} due to exception: {}", new Object[]{REL_FAILURE.getName(), e}, e);
+ requestFlowFile = session.penalize(requestFlowFile);
+ // transfer original to failure
+ session.transfer(requestFlowFile, REL_FAILURE);
+ } else {
+ logger.error("Yielding processor due to exception encountered as a source processor: {}", e);
+ context.yield();
}
- }
- private void transfer() throws IOException {
- // check if we should penalize the request
- if (!isSuccess()) {
- request = session.penalize(request);
+ // cleanup response flowfile, if applicable
+ try {
+ if (responseFlowFile != null) {
+ session.remove(responseFlowFile);
+ }
+ } catch (final Exception e1) {
+ logger.error("Could not cleanup response flowfile due to exception: {}", new Object[]{e1}, e1);
}
+ }
+ }
- // log the status codes from the response
- logger.info("Request to {} returned status code {} for {}",
- new Object[] {conn.getURL().toExternalForm(), statusCode, request});
- // transfer to the correct relationship
- // 2xx -> SUCCESS
- if (isSuccess()) {
- // we have two flowfiles to transfer
- session.transfer(request, REL_SUCCESS_REQ);
- session.transfer(response, REL_SUCCESS_RESP);
+ private Request configureRequest(final ProcessContext context, final ProcessSession session, final FlowFile requestFlowFile, URL url) {
+ Request.Builder requestBuilder = new Request.Builder();
- // 5xx -> RETRY
- } else if (statusCode / 100 == 5) {
- session.transfer(request, REL_RETRY);
+ requestBuilder = requestBuilder.url(url);
+ final String authUser = trimToEmpty(context.getProperty(PROP_BASIC_AUTH_USERNAME).getValue());
- // 1xx, 3xx, 4xx -> NO RETRY
- } else {
- session.transfer(request, REL_NO_RETRY);
- }
+ // If the username/password properties are set then check if digest auth is being used
+ if (!authUser.isEmpty() && "false".equalsIgnoreCase(context.getProperty(PROP_DIGEST_AUTH).getValue())) {
+ final String authPass = trimToEmpty(context.getProperty(PROP_BASIC_AUTH_PASSWORD).getValue());
+ String credential = com.squareup.okhttp.Credentials.basic(authUser, authPass);
+ requestBuilder = requestBuilder.header("Authorization", credential);
}
- private void setRequestProperties() {
+ // set the request method
+ String method = trimToEmpty(context.getProperty(PROP_METHOD).evaluateAttributeExpressions(requestFlowFile).getValue()).toUpperCase();
+ switch (method) {
+ case "GET":
+ requestBuilder = requestBuilder.get();
+ break;
+ case "POST":
+ RequestBody requestBody = getRequestBodyToSend(session, requestFlowFile);
+ requestBuilder = requestBuilder.post(requestBody);
+ break;
+ case "PUT":
+ requestBody = getRequestBodyToSend(session, requestFlowFile);
+ requestBuilder = requestBuilder.put(requestBody);
+ break;
+ case "HEAD":
+ requestBuilder = requestBuilder.head();
+ break;
+ case "DELETE":
+ requestBuilder = requestBuilder.delete();
+ break;
+ default:
+ requestBuilder = requestBuilder.method(method, null);
+ }
- // check if we should send the a Date header with the request
- if (context.getProperty(PROP_DATE_HEADER).asBoolean()) {
- conn.setRequestProperty("Date", getDateValue());
- }
+ requestBuilder = setHeaderProperties(context, requestBuilder, requestFlowFile);
- // iterate through the flowfile attributes, adding any attribute that
- // matches the attributes-to-send pattern. if the pattern is not set
- // (it's an optional property), ignore that attribute entirely
- if (attributesToSend != null) {
- Map<String, String> attributes = request.getAttributes();
- Matcher m = attributesToSend.matcher("");
- for (Map.Entry<String, String> entry : attributes.entrySet()) {
- String key = trimToEmpty(entry.getKey());
- String val = trimToEmpty(entry.getValue());
-
- // don't include any of the ignored attributes
- if (IGNORED_ATTRIBUTES.contains(key)) {
- continue;
- }
+ return requestBuilder.build();
+ }
- // check if our attribute key matches the pattern
- // if so, include in the request as a header
- m.reset(key);
- if (m.matches()) {
- conn.setRequestProperty(key, val);
- }
- }
+ private RequestBody getRequestBodyToSend(final ProcessSession session, final FlowFile requestFlowFile) {
+ return new RequestBody() {
+ @Override
+ public MediaType contentType() {
+ final String attributeValue = requestFlowFile.getAttribute(CoreAttributes.MIME_TYPE.key());
+ String contentType = attributeValue == null ? DEFAULT_CONTENT_TYPE : attributeValue;
+ return MediaType.parse(contentType);
}
+
+ @Override
+ public void writeTo(BufferedSink sink) throws IOException {
+ session.exportTo(requestFlowFile, sink.outputStream());
+ }
+ };
+ }
+
+ private Request.Builder setHeaderProperties(final ProcessContext context, Request.Builder requestBuilder, final FlowFile requestFlowFile) {
+ // check if we should send the a Date header with the request
+ if (context.getProperty(PROP_DATE_HEADER).asBoolean()) {
+ requestBuilder = requestBuilder.addHeader("Date", DATE_FORMAT.print(System.currentTimeMillis()));
+ }
+
+ for (String headerKey : dynamicPropertyNames) {
+ String headerValue = context.getProperty(headerKey).evaluateAttributeExpressions(requestFlowFile).getValue();
+ requestBuilder = requestBuilder.addHeader(headerKey, headerValue);
}
- /**
- * Returns a Map of flowfile attributes from the response http headers. Multivalue headers are naively converted to comma separated strings.
- */
- private Map<String, String> convertAttributesFromHeaders() throws IOException {
- // create a new hashmap to store the values from the connection
- Map<String, String> map = new HashMap<>();
- for (Map.Entry<String, List<String>> entry : conn.getHeaderFields().entrySet()) {
- String key = entry.getKey();
- if (key == null) {
+ // iterate through the flowfile attributes, adding any attribute that
+ // matches the attributes-to-send pattern. if the pattern is not set
+ // (it's an optional property), ignore that attribute entirely
+ if (regexAttributesToSend != null) {
+ Map<String, String> attributes = requestFlowFile.getAttributes();
+ Matcher m = regexAttributesToSend.matcher("");
+ for (Map.Entry<String, String> entry : attributes.entrySet()) {
+ String headerKey = trimToEmpty(entry.getKey());
+
+ // don't include any of the ignored attributes
+ if (IGNORED_ATTRIBUTES.contains(headerKey)) {
continue;
}
- List<String> values = entry.getValue();
-
- // we ignore any headers with no actual values (rare)
- if (values == null || values.isEmpty()) {
- continue;
+ // check if our attribute key matches the pattern
+ // if so, include in the request as a header
+ m.reset(headerKey);
+ if (m.matches()) {
+ String headerVal = trimToEmpty(entry.getValue());
+ requestBuilder = requestBuilder.addHeader(headerKey, headerVal);
}
+ }
+ }
+ return requestBuilder;
+ }
- // create a comma separated string from the values, this is stored in the map
- String value = csv(values);
- // put the csv into the map
- map.put(key, value);
+ private void route(FlowFile request, FlowFile response, ProcessSession session, ProcessContext context, int statusCode){
+ // check if we should penalize the request
+ if (!isSuccess(statusCode)) {
+ if (request == null) {
+ context.yield();
+ } else {
+ request = session.penalize(request);
}
+ }
- if (conn instanceof HttpsURLConnection) {
- HttpsURLConnection sconn = (HttpsURLConnection) conn;
- // this should seemingly not be required, but somehow the state of the jdk client is messed up
- // when retrieving SSL certificate related information if connect() has not been called previously.
- sconn.connect();
- map.put(REMOTE_DN, sconn.getPeerPrincipal().getName());
+ // If the property to output the response flowfile regardless of status code is set then transfer it
+ boolean responseSent = false;
+ if (context.getProperty(PROP_OUTPUT_RESPONSE_REGARDLESS).asBoolean()) {
+ session.transfer(response, REL_RESPONSE);
+ responseSent = true;
+ }
+
+ // transfer to the correct relationship
+ // 2xx -> SUCCESS
+ if (isSuccess(statusCode)) {
+ // we have two flowfiles to transfer
+ if (request != null) {
+ session.transfer(request, REL_SUCCESS_REQ);
+ }
+ if (response != null && !responseSent) {
+ session.transfer(response, REL_RESPONSE);
}
- return map;
- }
+ // 5xx -> RETRY
+ } else if (statusCode / 100 == 5) {
+ if (request != null) {
+ session.transfer(request, REL_RETRY);
+ }
- private boolean isSuccess() throws IOException {
- if (statusCode == 0) {
- throw new IllegalStateException("Status code unknown, connection hasn't been attempted.");
+ // 1xx, 3xx, 4xx -> NO RETRY
+ } else {
+ if (request != null) {
+ session.transfer(request, REL_NO_RETRY);
}
- return statusCode / 100 == 2;
}
- private void logRequest() {
- logger.debug("\nRequest to remote service:\n\t{}\n{}",
- new Object[] {conn.getURL().toExternalForm(), getLogString(conn.getRequestProperties())});
- }
+ }
- private void logResponse() {
- logger.debug("\nResponse from remote service:\n\t{}\n{}",
- new Object[] {conn.getURL().toExternalForm(), getLogString(conn.getHeaderFields())});
- }
+ private boolean isSuccess(int statusCode) {
+ return statusCode / 100 == 2;
+ }
- private String getLogString(Map<String, List<String>> map) {
- StringBuilder sb = new StringBuilder();
- for (Map.Entry<String, List<String>> entry : map.entrySet()) {
- List<String> list = entry.getValue();
- if (list.isEmpty()) {
- continue;
- }
- sb.append("\t");
- sb.append(entry.getKey());
- sb.append(": ");
- if (list.size() == 1) {
- sb.append(list.get(0));
- } else {
- sb.append(list.toString());
- }
- sb.append("\n");
- }
- return sb.toString();
- }
+ private void logRequest(ProcessorLog logger, Request request) {
+ logger.debug("\nRequest to remote service:\n\t{}\n{}",
+ new Object[]{request.url().toExternalForm(), getLogString(request.headers().toMultimap())});
+ }
- /**
- * Convert a collection of string values into a overly simple comma separated string.
- *
- * Does not handle the case where the value contains the delimiter. i.e. if a value contains a comma, this method does nothing to try and escape or quote the value, in traditional csv style.
- */
- private String csv(Collection<String> values) {
- if (values == null || values.isEmpty()) {
- return "";
- }
- if (values.size() == 1) {
- return values.iterator().next();
- }
+ private void logResponse(ProcessorLog logger, URL url, Response response) {
+ logger.debug("\nResponse from remote service:\n\t{}\n{}",
+ new Object[]{url.toExternalForm(), getLogString(response.headers().toMultimap())});
+ }
- StringBuilder sb = new StringBuilder();
- for (String value : values) {
- value = value.trim();
- if (value.isEmpty()) {
- continue;
- }
- if (sb.length() > 0) {
- sb.append(", ");
- }
- sb.append(value);
+ private String getLogString(Map<String, List<String>> map) {
+ StringBuilder sb = new StringBuilder();
+ for (Map.Entry<String, List<String>> entry : map.entrySet()) {
+ List<String> list = entry.getValue();
+ if (list.isEmpty()) {
+ continue;
+ }
+ sb.append("\t");
+ sb.append(entry.getKey());
+ sb.append(": ");
+ if (list.size() == 1) {
+ sb.append(list.get(0));
+ } else {
+ sb.append(list.toString());
}
- return sb.toString().trim();
+ sb.append("\n");
}
+ return sb.toString();
+ }
- /**
- * Return the current datetime as an RFC 1123 formatted string in the GMT tz.
- */
- private String getDateValue() {
- return dateFormat.print(System.currentTimeMillis());
+ /**
+ * Convert a collection of string values into a overly simple comma separated string.
+ * <p/>
+ * Does not handle the case where the value contains the delimiter. i.e. if a value contains a comma, this method does nothing to try and escape or quote the value, in traditional csv style.
+ */
+ private String csv(Collection<String> values) {
+ if (values == null || values.isEmpty()) {
+ return "";
+ }
+ if (values.size() == 1) {
+ return values.iterator().next();
}
- /**
- * Returns a string from the input stream using the specified character encoding.
- */
- private String toString(InputStream is, Charset charset) throws IOException {
- if (is == null) {
- return "";
+ StringBuilder sb = new StringBuilder();
+ for (String value : values) {
+ value = value.trim();
+ if (value.isEmpty()) {
+ continue;
}
-
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- byte[] buf = new byte[4096];
- int len;
- while ((len = is.read(buf)) != -1) {
- out.write(buf, 0, len);
+ if (sb.length() > 0) {
+ sb.append(", ");
}
- return new String(out.toByteArray(), charset);
+ sb.append(value);
}
+ return sb.toString().trim();
+ }
- /**
- * Returns the input stream to use for reading from the remote server. We're either going to want the inputstream or errorstream, effectively depending on the status code.
- * <p>
- * This method can return null if there is no inputstream to read from. For example, if the remote server did not send a message body. eg. 204 No Content or 304 Not Modified
- */
- private InputStream getResponseStream() {
- try {
- InputStream is = conn.getErrorStream();
- if (is == null) {
- is = conn.getInputStream();
- }
- return new BufferedInputStream(is);
+ /**
+ * Returns a Map of flowfile attributes from the response http headers. Multivalue headers are naively converted to comma separated strings.
+ */
+ private Map<String, String> convertAttributesFromHeaders(URL url, Response responseHttp){
+ // create a new hashmap to store the values from the connection
+ Map<String, String> map = new HashMap<>();
+ for (Map.Entry<String, List<String>> entry : responseHttp.headers().toMultimap().entrySet()) {
+ String key = entry.getKey();
+ if (key == null) {
+ continue;
+ }
- } catch (IOException e) {
- logger.warn("Response stream threw an exception: {}", new Object[] {e}, e);
- return null;
+ List<String> values = entry.getValue();
+
+ // we ignore any headers with no actual values (rare)
+ if (values == null || values.isEmpty()) {
+ continue;
}
+
+ // create a comma separated string from the values, this is stored in the map
+ String value = csv(values);
+
+ // put the csv into the map
+ map.put(key, value);
}
- /**
- * Writes the status attributes onto the flowfile, returning the flowfile that was updated.
- */
- private FlowFile writeStatusAttributes(FlowFile flowfile) {
- flowfile = session.putAttribute(flowfile, STATUS_CODE, String.valueOf(statusCode));
- flowfile = session.putAttribute(flowfile, STATUS_MESSAGE, statusMessage);
- flowfile = session.putAttribute(flowfile, REQUEST_URL, conn.getURL().toExternalForm());
- flowfile = session.putAttribute(flowfile, TRANSACTION_ID, Long.toString(txId));
- return flowfile;
+ if ("HTTPS".equals(url.getProtocol().toUpperCase())) {
+ map.put(REMOTE_DN, responseHttp.handshake().peerPrincipal().getName());
}
- /**
- *
- */
- private static class OverrideHostnameVerifier implements HostnameVerifier {
+ return map;
+ }
- private final String trustedHostname;
- private final HostnameVerifier delegate;
+ private Charset getCharsetFromMediaType(MediaType contentType) {
+ return contentType != null ? contentType.charset(StandardCharsets.UTF_8) : StandardCharsets.UTF_8;
+ }
- private OverrideHostnameVerifier(String trustedHostname, HostnameVerifier delegate) {
- this.trustedHostname = trustedHostname;
- this.delegate = delegate;
- }
+ private static class OverrideHostnameVerifier implements HostnameVerifier {
- @Override
- public boolean verify(String hostname, SSLSession session) {
- if (trustedHostname.equalsIgnoreCase(hostname)) {
- return true;
- }
- return delegate.verify(hostname, session);
+ private final String trustedHostname;
+ private final HostnameVerifier delegate;
+
+ private OverrideHostnameVerifier(String trustedHostname, HostnameVerifier delegate) {
+ this.trustedHostname = trustedHostname;
+ this.delegate = delegate;
+ }
+
+ @Override
+ public boolean verify(String hostname, SSLSession session) {
+ if (trustedHostname.equalsIgnoreCase(hostname)) {
+ return true;
}
+ return delegate.verify(hostname, session);
}
}
-
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/8c2323dc/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestInvokeHTTP.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestInvokeHTTP.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestInvokeHTTP.java
index a26b2ed..a82bc5a 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestInvokeHTTP.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestInvokeHTTP.java
@@ -19,22 +19,27 @@ package org.apache.nifi.processors.standard;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.URL;
+import java.util.HashMap;
+import java.util.Map;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.nifi.processors.standard.util.TestInvokeHttpCommon;
+import org.apache.nifi.ssl.StandardSSLContextService;
import org.apache.nifi.util.MockFlowFile;
import org.apache.nifi.util.TestRunners;
import org.eclipse.jetty.server.Request;
import org.eclipse.jetty.server.handler.AbstractHandler;
import org.junit.After;
import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
+
public class TestInvokeHTTP extends TestInvokeHttpCommon {
@BeforeClass
@@ -72,41 +77,99 @@ public class TestInvokeHTTP extends TestInvokeHttpCommon {
return new TestServer();
}
+ @Test
+ public void testSslSetHttpRequest() throws Exception {
+
+ final Map<String, String> sslProperties = new HashMap<>();
+ sslProperties.put(StandardSSLContextService.KEYSTORE.getName(), "src/test/resources/localhost-ks.jks");
+ sslProperties.put(StandardSSLContextService.KEYSTORE_PASSWORD.getName(), "localtest");
+ sslProperties.put(StandardSSLContextService.KEYSTORE_TYPE.getName(), "JKS");
+ sslProperties.put(StandardSSLContextService.TRUSTSTORE.getName(), "src/test/resources/localhost-ts.jks");
+ sslProperties.put(StandardSSLContextService.TRUSTSTORE_PASSWORD.getName(), "localtest");
+ sslProperties.put(StandardSSLContextService.TRUSTSTORE_TYPE.getName(), "JKS");
+
+ runner = TestRunners.newTestRunner(InvokeHTTP.class);
+ final StandardSSLContextService sslService = new StandardSSLContextService();
+ runner.addControllerService("ssl-context", sslService, sslProperties);
+ runner.enableControllerService(sslService);
+ runner.setProperty(InvokeHTTP.PROP_SSL_CONTEXT_SERVICE, "ssl-context");
+
+ addHandler(new GetOrHeadHandler());
+
+ runner.setProperty(InvokeHTTP.PROP_URL, url + "/status/200");
+
+ createFlowFiles(runner);
+
+ runner.run();
+
+ runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
+
+ // expected in request status.code and status.message
+ // original flow file (+attributes)
+ final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.REL_SUCCESS_REQ).get(0);
+ bundle.assertContentEquals("Hello".getBytes("UTF-8"));
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
+ bundle.assertAttributeEquals("Foo", "Bar");
+
+ // expected in response
+ // status code, status message, all headers from server response --> ff attributes
+ // server response message body into payload of ff
+ final MockFlowFile bundle1 = runner.getFlowFilesForRelationship(InvokeHTTP.REL_RESPONSE).get(0);
+ bundle1.assertContentEquals("/status/200".getBytes("UTF-8"));
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
+ bundle1.assertAttributeEquals("Foo", "Bar");
+ bundle1.assertAttributeEquals("Content-Type", "text/plain; charset=ISO-8859-1");
+ bundle1.assertAttributeEquals("OkHttp-Selected-Protocol", "http/1.1");
+ }
+
// Currently InvokeHttp does not support Proxy via Https
@Test
public void testProxy() throws Exception {
addHandler(new MyProxyHandler());
URL proxyURL = new URL(url);
- runner.setProperty(InvokeHTTP.Config.PROP_URL, "http://nifi.apache.org/"); // just a dummy URL no connection goes out
- runner.setProperty(InvokeHTTP.Config.PROP_PROXY_HOST, proxyURL.getHost());
- runner.setProperty(InvokeHTTP.Config.PROP_PROXY_PORT, String.valueOf(proxyURL.getPort()));
+ runner.setProperty(InvokeHTTP.PROP_URL, "http://nifi.apache.org/"); // just a dummy URL no connection goes out
+ runner.setProperty(InvokeHTTP.PROP_PROXY_HOST, proxyURL.getHost());
+
+ try{
+ runner.run();
+ Assert.fail();
+ } catch (AssertionError e){
+ // Expect assetion error when proxy port isn't set but host is.
+ }
+ runner.setProperty(InvokeHTTP.PROP_PROXY_PORT, String.valueOf(proxyURL.getPort()));
createFlowFiles(runner);
runner.run();
- runner.assertTransferCount(InvokeHTTP.Config.REL_SUCCESS_REQ, 1);
- runner.assertTransferCount(InvokeHTTP.Config.REL_SUCCESS_RESP, 1);
- runner.assertTransferCount(InvokeHTTP.Config.REL_RETRY, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_NO_RETRY, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_FAILURE, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
//expected in request status.code and status.message
//original flow file (+attributes)
- final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.Config.REL_SUCCESS_REQ).get(0);
+ final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.REL_SUCCESS_REQ).get(0);
bundle.assertContentEquals("Hello".getBytes("UTF-8"));
- bundle.assertAttributeEquals(InvokeHTTP.Config.STATUS_CODE, "200");
- bundle.assertAttributeEquals(InvokeHTTP.Config.STATUS_MESSAGE, "OK");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
bundle.assertAttributeEquals("Foo", "Bar");
//expected in response
//status code, status message, all headers from server response --> ff attributes
//server response message body into payload of ff
- final MockFlowFile bundle1 = runner.getFlowFilesForRelationship(InvokeHTTP.Config.REL_SUCCESS_RESP).get(0);
+ final MockFlowFile bundle1 = runner.getFlowFilesForRelationship(InvokeHTTP.REL_RESPONSE).get(0);
bundle1.assertContentEquals("http://nifi.apache.org/".getBytes("UTF-8"));
- bundle1.assertAttributeEquals(InvokeHTTP.Config.STATUS_CODE, "200");
- bundle1.assertAttributeEquals(InvokeHTTP.Config.STATUS_MESSAGE, "OK");
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
bundle1.assertAttributeEquals("Foo", "Bar");
bundle1.assertAttributeEquals("Content-Type", "text/plain; charset=ISO-8859-1");
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/8c2323dc/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestInvokeHttpSSL.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestInvokeHttpSSL.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestInvokeHttpSSL.java
index d155b74..b3cd9dc 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestInvokeHttpSSL.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestInvokeHttpSSL.java
@@ -17,7 +17,6 @@
package org.apache.nifi.processors.standard;
-import org.apache.nifi.processors.standard.InvokeHTTP.Config;
import org.apache.nifi.processors.standard.util.TestInvokeHttpCommon;
import org.apache.nifi.ssl.StandardSSLContextService;
import org.apache.nifi.util.TestRunners;
@@ -63,7 +62,7 @@ public class TestInvokeHttpSSL extends TestInvokeHttpCommon {
final StandardSSLContextService sslService = new StandardSSLContextService();
runner.addControllerService("ssl-context", sslService, sslProperties);
runner.enableControllerService(sslService);
- runner.setProperty(Config.PROP_SSL_CONTEXT_SERVICE, "ssl-context");
+ runner.setProperty(InvokeHTTP.PROP_SSL_CONTEXT_SERVICE, "ssl-context");
server.clearHandlers();
}
[13/50] [abbrv] nifi git commit: Merge branch 'NIFI-748' of
https://github.com/olegz/nifi into NIFI-748
Posted by mc...@apache.org.
Merge branch 'NIFI-748' of https://github.com/olegz/nifi into NIFI-748
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/453b140d
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/453b140d
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/453b140d
Branch: refs/heads/NIFI-655
Commit: 453b140d6bbd73bc63fa3e8cb9e7072eb7709e50
Parents: d16392c 15880f9
Author: Mark Payne <ma...@hotmail.com>
Authored: Tue Nov 17 09:23:10 2015 -0500
Committer: Mark Payne <ma...@hotmail.com>
Committed: Tue Nov 17 09:23:10 2015 -0500
----------------------------------------------------------------------
.../PersistentProvenanceRepository.java | 1 +
.../nifi/provenance/lucene/DocsReader.java | 115 ++++++-------------
.../nifi/provenance/lucene/IndexSearch.java | 2 +-
.../nifi/provenance/lucene/LineageQuery.java | 2 +-
.../nifi/provenance/lucene/LuceneUtil.java | 36 +++++-
.../TestPersistentProvenanceRepository.java | 70 +++++++++++
6 files changed, 144 insertions(+), 82 deletions(-)
----------------------------------------------------------------------
[46/50] [abbrv] nifi git commit: NIFI-655: - Refactoring web security
to use Spring Security Java Configuration. - Introducing security in Web UI
in order to get JWT.
Posted by mc...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java
index bd7482e..e7a3328 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java
@@ -170,6 +170,7 @@ import org.apache.nifi.web.api.dto.status.ClusterProcessGroupStatusDTO;
import org.apache.nifi.web.api.dto.status.NodeProcessGroupStatusDTO;
import org.apache.nifi.web.dao.ControllerServiceDAO;
import org.apache.nifi.web.dao.ReportingTaskDAO;
+import org.apache.nifi.web.security.user.NewAccountRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.security.access.AccessDeniedException;
@@ -762,7 +763,7 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade {
if (user == null) {
throw new WebApplicationException(new Throwable("Unable to access details for current user."));
}
- final String userDn = user.getDn();
+ final String userDn = user.getIdentity();
if (Node.Status.CONNECTING.name().equalsIgnoreCase(nodeDTO.getStatus())) {
clusterManager.requestReconnection(nodeDTO.getNodeId(), userDn);
@@ -1782,7 +1783,7 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade {
// create a purge action to record that records are being removed
FlowChangeAction purgeAction = new FlowChangeAction();
- purgeAction.setUserIdentity(user.getDn());
+ purgeAction.setUserIdentity(user.getIdentity());
purgeAction.setUserName(user.getUserName());
purgeAction.setOperation(Operation.Purge);
purgeAction.setTimestamp(new Date());
@@ -1820,6 +1821,23 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade {
}
@Override
+ public UserDTO createUser() {
+ NewAccountRequest newAccountRequest = NiFiUserUtils.getNewAccountRequest();
+
+ // log the new user account request
+ logger.info("Requesting new user account for " + newAccountRequest.getUsername());
+
+ // get the justification
+ String justification = newAccountRequest.getJustification();
+ if (justification == null) {
+ justification = StringUtils.EMPTY;
+ }
+
+ // create the pending user account
+ return dtoFactory.createUserDTO(userService.createPendingUserAccount(newAccountRequest.getUsername(), justification));
+ }
+
+ @Override
public UserDTO updateUser(UserDTO userDto) {
NiFiUser user;
@@ -2256,7 +2274,7 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade {
}
final Set<String> allowedUsers = port.getUserAccessControl();
- if (allowedUsers.contains(user.getDn())) {
+ if (allowedUsers.contains(user.getIdentity())) {
return true;
}
@@ -2632,7 +2650,7 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade {
final UserDTO groupedUser = groupedUserDTOs.get(user.getUserGroup());
groupedUser.setId(groupedUser.getId() + "," + String.valueOf(user.getId()));
groupedUser.setUserName(groupedUser.getUserName() + ", " + user.getUserName());
- groupedUser.setDn(groupedUser.getDn() + ", " + user.getDn());
+ groupedUser.setDn(groupedUser.getDn() + ", " + user.getIdentity());
groupedUser.setCreation(getOldestDate(groupedUser.getCreation(), user.getCreation()));
groupedUser.setLastAccessed(getNewestDate(groupedUser.getLastAccessed(), user.getLastAccessed()));
groupedUser.setLastVerified(getNewestDate(groupedUser.getLastVerified(), user.getLastVerified()));
@@ -2731,7 +2749,7 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade {
throw new WebApplicationException(new Throwable("Unable to access details for current user."));
}
- final String userDn = user.getDn();
+ final String userDn = user.getIdentity();
clusterManager.deleteNode(nodeId, userDn);
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiWebConfigurationContext.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiWebConfigurationContext.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiWebConfigurationContext.java
index 8e56fc2..9502bf2 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiWebConfigurationContext.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiWebConfigurationContext.java
@@ -158,19 +158,19 @@ public class StandardNiFiWebConfigurationContext implements NiFiWebConfiguration
@Override
public String getCurrentUserDn() {
- String userDn = NiFiUser.ANONYMOUS_USER_DN;
+ String userIdentity = NiFiUser.ANONYMOUS_USER_IDENTITY;
final NiFiUser user = NiFiUserUtils.getNiFiUser();
if (user != null) {
- userDn = user.getDn();
+ userIdentity = user.getIdentity();
}
- return userDn;
+ return userIdentity;
}
@Override
public String getCurrentUserName() {
- String userName = NiFiUser.ANONYMOUS_USER_DN;
+ String userName = NiFiUser.ANONYMOUS_USER_IDENTITY;
final NiFiUser user = NiFiUserUtils.getNiFiUser();
if (user != null) {
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiWebContext.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiWebContext.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiWebContext.java
index 03ce630..7e9e77e 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiWebContext.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiWebContext.java
@@ -129,19 +129,19 @@ public class StandardNiFiWebContext implements NiFiWebContext {
@Override
public String getCurrentUserDn() {
- String userDn = NiFiUser.ANONYMOUS_USER_DN;
+ String userIdentity = NiFiUser.ANONYMOUS_USER_IDENTITY;
final NiFiUser user = NiFiUserUtils.getNiFiUser();
if (user != null) {
- userDn = user.getDn();
+ userIdentity = user.getIdentity();
}
- return userDn;
+ return userIdentity;
}
@Override
public String getCurrentUserName() {
- String userName = NiFiUser.ANONYMOUS_USER_DN;
+ String userName = NiFiUser.ANONYMOUS_USER_IDENTITY;
final NiFiUser user = NiFiUserUtils.getNiFiUser();
if (user != null) {
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/AccessResource.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/AccessResource.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/AccessResource.java
new file mode 100644
index 0000000..e198438
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/AccessResource.java
@@ -0,0 +1,424 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.api;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
+import io.jsonwebtoken.JwtException;
+import org.apache.nifi.util.NiFiProperties;
+import com.wordnik.swagger.annotations.Api;
+import com.wordnik.swagger.annotations.ApiOperation;
+import com.wordnik.swagger.annotations.ApiParam;
+import com.wordnik.swagger.annotations.ApiResponse;
+import com.wordnik.swagger.annotations.ApiResponses;
+import java.net.URI;
+import java.security.cert.X509Certificate;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.FormParam;
+import javax.ws.rs.POST;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.admin.service.AdministrationException;
+import org.apache.nifi.authentication.AuthenticationResponse;
+import org.apache.nifi.authentication.LoginCredentials;
+import org.apache.nifi.authentication.LoginIdentityProvider;
+import org.apache.nifi.authentication.exception.IdentityAccessException;
+import org.apache.nifi.authentication.exception.InvalidLoginCredentialsException;
+import org.apache.nifi.security.util.CertificateUtils;
+import org.apache.nifi.web.api.dto.AccessStatusDTO;
+import org.apache.nifi.web.api.dto.AccessConfigurationDTO;
+import org.apache.nifi.web.api.dto.RevisionDTO;
+import org.apache.nifi.web.api.entity.AccessStatusEntity;
+import org.apache.nifi.web.api.entity.AccessConfigurationEntity;
+import org.apache.nifi.web.api.request.ClientIdParameter;
+import org.apache.nifi.web.security.InvalidAuthenticationException;
+import org.apache.nifi.web.security.ProxiedEntitiesUtils;
+import org.apache.nifi.web.security.UntrustedProxyException;
+import org.apache.nifi.web.security.jwt.JwtService;
+import org.apache.nifi.web.security.token.LoginAuthenticationToken;
+import org.apache.nifi.web.security.token.NiFiAuthenticationRequestToken;
+import org.apache.nifi.web.security.x509.X509CertificateExtractor;
+import org.apache.nifi.web.security.x509.X509IdentityProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.security.access.AccessDeniedException;
+import org.springframework.security.authentication.AccountStatusException;
+import org.springframework.security.authentication.AuthenticationServiceException;
+import org.springframework.security.core.AuthenticationException;
+import org.springframework.security.core.userdetails.AuthenticationUserDetailsService;
+import org.springframework.security.core.userdetails.UsernameNotFoundException;
+
+/**
+ * RESTful endpoint for managing a cluster.
+ */
+@Path("/access")
+@Api(
+ value = "/access",
+ description = "Endpoints for obtaining an access token or checking access status"
+)
+public class AccessResource extends ApplicationResource {
+
+ private static final Logger logger = LoggerFactory.getLogger(AccessResource.class);
+
+ private static final String AUTHORIZATION = "Authorization";
+
+ private NiFiProperties properties;
+
+ private LoginIdentityProvider loginIdentityProvider;
+ private X509CertificateExtractor certificateExtractor;
+ private X509IdentityProvider certificateIdentityProvider;
+ private JwtService jwtService;
+
+ private AuthenticationUserDetailsService<NiFiAuthenticationRequestToken> userDetailsService;
+
+ /**
+ * Retrieves the access configuration for this NiFi.
+ *
+ * @param httpServletRequest the servlet request
+ * @param clientId Optional client id. If the client id is not specified, a new one will be generated. This value (whether specified or generated) is included in the response.
+ * @return A accessConfigurationEntity
+ */
+ @GET
+ @Consumes(MediaType.WILDCARD)
+ @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML})
+ @Path("/config")
+ @ApiOperation(
+ value = "Retrieves the access configuration for this NiFi",
+ response = AccessConfigurationEntity.class
+ )
+ public Response getLoginConfig(
+ @Context HttpServletRequest httpServletRequest,
+ @ApiParam(
+ value = "If the client id is not specified, new one will be generated. This value (whether specified or generated) is included in the response.",
+ required = false
+ )
+ @QueryParam(CLIENT_ID) @DefaultValue(StringUtils.EMPTY) ClientIdParameter clientId) {
+
+ final AccessConfigurationDTO accessConfiguration = new AccessConfigurationDTO();
+
+ // specify whether login should be supported and only support for secure requests
+ accessConfiguration.setSupportsLogin(loginIdentityProvider != null && httpServletRequest.isSecure());
+ accessConfiguration.setSupportsAnonymous(!properties.getAnonymousAuthorities().isEmpty() || !httpServletRequest.isSecure());
+
+ // create the revision
+ final RevisionDTO revision = new RevisionDTO();
+ revision.setClientId(clientId.getClientId());
+
+ // create the response entity
+ final AccessConfigurationEntity entity = new AccessConfigurationEntity();
+ entity.setRevision(revision);
+ entity.setConfig(accessConfiguration);
+
+ // generate the response
+ return clusterContext(generateOkResponse(entity)).build();
+ }
+
+ /**
+ * Gets the status the client's access.
+ *
+ * @param httpServletRequest the servlet request
+ * @param clientId Optional client id. If the client id is not specified, a new one will be generated. This value (whether specified or generated) is included in the response.
+ * @return A accessStatusEntity
+ */
+ @GET
+ @Consumes(MediaType.WILDCARD)
+ @Produces({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON})
+ @Path("")
+ @ApiOperation(
+ value = "Gets the status the client's access",
+ response = AccessStatusEntity.class
+ )
+ @ApiResponses(
+ value = {
+ @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."),
+ @ApiResponse(code = 401, message = "Unable to determine access status because the client could not be authenticated."),
+ @ApiResponse(code = 403, message = "Unable to determine access status because the client is not authorized to make this request."),
+ @ApiResponse(code = 409, message = "Unable to determine access status because NiFi is not in the appropriate state."),
+ @ApiResponse(code = 500, message = "Unable to determine access status because an unexpected error occurred.")
+ }
+ )
+ public Response getAccessStatus(
+ @Context HttpServletRequest httpServletRequest,
+ @ApiParam(
+ value = "If the client id is not specified, new one will be generated. This value (whether specified or generated) is included in the response.",
+ required = false
+ )
+ @QueryParam(CLIENT_ID) @DefaultValue(StringUtils.EMPTY) ClientIdParameter clientId) {
+
+ // only consider user specific access over https
+ if (!httpServletRequest.isSecure()) {
+ throw new IllegalStateException("User authentication/authorization is only supported when running over HTTPS.");
+ }
+
+ final AccessStatusDTO accessStatus = new AccessStatusDTO();
+
+ try {
+ final X509Certificate[] certificates = certificateExtractor.extractClientCertificate(httpServletRequest);
+
+ // if there is not certificate, consider a token
+ if (certificates == null) {
+ // look for an authorization token
+ final String authorization = httpServletRequest.getHeader(AUTHORIZATION);
+
+ // if there is no authorization header, we don't know the user
+ if (authorization == null) {
+ accessStatus.setStatus(AccessStatusDTO.Status.UNKNOWN.name());
+ accessStatus.setMessage("No credentials supplied, unknown user.");
+ } else {
+ try {
+ // Extract the Base64 encoded token from the Authorization header
+ final String token = StringUtils.substringAfterLast(authorization, " ");
+ final String principal = jwtService.getAuthenticationFromToken(token);
+
+ // set the user identity
+ accessStatus.setIdentity(principal);
+ accessStatus.setUsername(CertificateUtils.extractUsername(principal));
+
+ // without a certificate, this is not a proxied request
+ final List<String> chain = Arrays.asList(principal);
+
+ // check authorization for this user
+ checkAuthorization(chain);
+
+ // no issues with authorization
+ accessStatus.setStatus(AccessStatusDTO.Status.ACTIVE.name());
+ accessStatus.setMessage("Account is active and authorized");
+ } catch (JwtException e) {
+ throw new InvalidAuthenticationException(e.getMessage(), e);
+ }
+ }
+ } else {
+ try {
+ final AuthenticationResponse authenticationResponse = certificateIdentityProvider.authenticate(certificates);
+
+ // get the proxy chain and ensure its populated
+ final List<String> proxyChain = ProxiedEntitiesUtils.buildProxiedEntitiesChain(httpServletRequest, authenticationResponse.getIdentity());
+ if (proxyChain.isEmpty()) {
+ logger.error(String.format("Unable to parse the proxy chain %s from the incoming request.", authenticationResponse.getIdentity()));
+ throw new IllegalArgumentException("Unable to determine the user from the incoming request.");
+ }
+
+ // set the user identity
+ accessStatus.setIdentity(proxyChain.get(0));
+ accessStatus.setUsername(CertificateUtils.extractUsername(proxyChain.get(0)));
+
+ // ensure the proxy chain is authorized
+ checkAuthorization(proxyChain);
+
+ // no issues with authorization
+ accessStatus.setStatus(AccessStatusDTO.Status.ACTIVE.name());
+ accessStatus.setMessage("Account is active and authorized");
+ } catch (final IllegalArgumentException iae) {
+ throw new InvalidAuthenticationException(iae.getMessage(), iae);
+ }
+ }
+ } catch (final UsernameNotFoundException unfe) {
+ accessStatus.setStatus(AccessStatusDTO.Status.UNREGISTERED.name());
+ accessStatus.setMessage(String.format("Unregistered user %s", accessStatus.getIdentity()));
+ } catch (final AccountStatusException ase) {
+ accessStatus.setStatus(AccessStatusDTO.Status.NOT_ACTIVE.name());
+ accessStatus.setMessage(ase.getMessage());
+ } catch (final UntrustedProxyException upe) {
+ throw new AccessDeniedException(upe.getMessage(), upe);
+ } catch (final AuthenticationServiceException ase) {
+ throw new AdministrationException(ase.getMessage(), ase);
+ }
+
+ // create the revision
+ final RevisionDTO revision = new RevisionDTO();
+ revision.setClientId(clientId.getClientId());
+
+ // create the entity
+ final AccessStatusEntity entity = new AccessStatusEntity();
+ entity.setRevision(revision);
+ entity.setAccessStatus(accessStatus);
+
+ return generateOkResponse(entity).build();
+ }
+
+ /**
+ * Checks the status of the proxy.
+ *
+ * @param proxyChain the proxy chain
+ * @throws AuthenticationException if the proxy chain is not authorized
+ */
+ private void checkAuthorization(final List<String> proxyChain) throws AuthenticationException {
+ userDetailsService.loadUserDetails(new NiFiAuthenticationRequestToken(proxyChain));
+ }
+
+ /**
+ * Creates a token for accessing the REST API via username/password.
+ *
+ * @param httpServletRequest the servlet request
+ * @param username the username
+ * @param password the password
+ * @return A JWT (string)
+ */
+ @POST
+ @Consumes(MediaType.APPLICATION_FORM_URLENCODED)
+ @Produces(MediaType.TEXT_PLAIN)
+ @Path("/token")
+ @ApiOperation(
+ value = "Creates a token for accessing the REST API via username/password",
+ response = String.class
+ )
+ @ApiResponses(
+ value = {
+ @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."),
+ @ApiResponse(code = 403, message = "Client is not authorized to make this request."),
+ @ApiResponse(code = 409, message = "Unable to create access token because NiFi is not in the appropriate state. (i.e. may not be configured to support username/password login."),
+ @ApiResponse(code = 500, message = "Unable to create access token because an unexpected error occurred.")
+ }
+ )
+ public Response createAccessToken(
+ @Context HttpServletRequest httpServletRequest,
+ @FormParam("username") String username,
+ @FormParam("password") String password) {
+
+ // only support access tokens when communicating over HTTPS
+ if (!httpServletRequest.isSecure()) {
+ throw new IllegalStateException("Access tokens are only issued over HTTPS.");
+ }
+
+ // if not configuration for login, don't consider credentials
+ if (loginIdentityProvider == null) {
+ throw new IllegalStateException("Username/Password login not supported by this NiFi.");
+ }
+
+ final LoginAuthenticationToken loginAuthenticationToken;
+
+ final X509Certificate[] certificates = certificateExtractor.extractClientCertificate(httpServletRequest);
+
+ // if there is not certificate, consider login credentials
+ if (certificates == null) {
+ // ensure we have login credentials
+ if (StringUtils.isBlank(username) || StringUtils.isBlank(password)) {
+ throw new IllegalArgumentException("The username and password must be specified.");
+ }
+
+ try {
+ // attempt to authenticate
+ final AuthenticationResponse authenticationResponse = loginIdentityProvider.authenticate(new LoginCredentials(username, password));
+ final long maxExpiration = TimeUnit.MILLISECONDS.convert(12, TimeUnit.HOURS);
+ final long minExpiration = TimeUnit.MILLISECONDS.convert(1, TimeUnit.MINUTES);
+
+ long expiration = authenticationResponse.getExpiration();
+ if (expiration > maxExpiration) {
+ expiration = maxExpiration;
+
+ logger.warn(String.format("Max token expiration exceeded. Setting expiration to %s from %s for %s", expiration,
+ authenticationResponse.getExpiration(), authenticationResponse.getIdentity()));
+ } else if (expiration < minExpiration) {
+ expiration = minExpiration;
+
+ logger.warn(String.format("Min token expiration not met. Setting expiration to %s from %s for %s", expiration,
+ authenticationResponse.getExpiration(), authenticationResponse.getIdentity()));
+ }
+
+ // create the authentication token
+ loginAuthenticationToken = new LoginAuthenticationToken(authenticationResponse.getIdentity(), expiration, authenticationResponse.getIssuer());
+ } catch (final InvalidLoginCredentialsException ilce) {
+ throw new IllegalArgumentException("The supplied username and password are not valid.", ilce);
+ } catch (final IdentityAccessException iae) {
+ throw new AdministrationException(iae.getMessage(), iae);
+ }
+ } else {
+ // consider a certificate
+ final AuthenticationResponse authenticationResponse = certificateIdentityProvider.authenticate(certificates);
+
+ // get the proxy chain and ensure its populated
+ final List<String> proxyChain = ProxiedEntitiesUtils.buildProxiedEntitiesChain(httpServletRequest, authenticationResponse.getIdentity());
+ if (proxyChain.isEmpty()) {
+ logger.error(String.format("Unable to parse the proxy chain %s from the incoming request.", authenticationResponse.getIdentity()));
+ throw new IllegalArgumentException("Unable to determine the user from the incoming request.");
+ }
+
+ // authorize the proxy if necessary
+ authorizeProxyIfNecessary(proxyChain);
+
+ // create the authentication token
+ loginAuthenticationToken = new LoginAuthenticationToken(proxyChain.get(0), authenticationResponse.getExpiration(), authenticationResponse.getIssuer());
+ }
+
+ // generate JWT for response
+ final String token = jwtService.generateSignedToken(loginAuthenticationToken);
+
+ // build the response
+ final URI uri = URI.create(generateResourceUri("access", "token"));
+ return generateCreatedResponse(uri, token).build();
+ }
+
+ /**
+ * Ensures the proxyChain is authorized before allowing the user to be authenticated.
+ *
+ * @param proxyChain the proxy chain
+ * @throws AuthenticationException if the proxy chain is not authorized
+ */
+ private void authorizeProxyIfNecessary(final List<String> proxyChain) throws AuthenticationException {
+ if (proxyChain.size() > 1) {
+ try {
+ userDetailsService.loadUserDetails(new NiFiAuthenticationRequestToken(proxyChain));
+ } catch (final UsernameNotFoundException unfe) {
+ // if a username not found exception was thrown, the proxies were authorized and now
+ // we can issue a new token to the end user which they will use to identify themselves
+ // when they enter a new account request
+ } catch (final AuthenticationServiceException ase) {
+ // throw an administration exception which will return a 500
+ throw new AdministrationException(ase.getMessage(), ase);
+ } catch (final Exception e) {
+ // any other issue we're going to treat as access denied exception which will return 403
+ throw new AccessDeniedException(e.getMessage(), e);
+ }
+ }
+ }
+
+ // setters
+ public void setProperties(NiFiProperties properties) {
+ this.properties = properties;
+ }
+
+ public void setLoginIdentityProvider(LoginIdentityProvider loginIdentityProvider) {
+ this.loginIdentityProvider = loginIdentityProvider;
+ }
+
+ public void setJwtService(JwtService jwtService) {
+ this.jwtService = jwtService;
+ }
+
+ public void setCertificateExtractor(X509CertificateExtractor certificateExtractor) {
+ this.certificateExtractor = certificateExtractor;
+ }
+
+ public void setCertificateIdentityProvider(X509IdentityProvider certificateIdentityProvider) {
+ this.certificateIdentityProvider = certificateIdentityProvider;
+ }
+
+ public void setUserDetailsService(AuthenticationUserDetailsService<NiFiAuthenticationRequestToken> userDetailsService) {
+ this.userDetailsService = userDetailsService;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ApplicationResource.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ApplicationResource.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ApplicationResource.java
index 3bad5e3..d0c36d4 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ApplicationResource.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ApplicationResource.java
@@ -23,7 +23,6 @@ import com.sun.jersey.server.impl.model.method.dispatch.FormDispatchProvider;
import java.io.Serializable;
import java.net.URI;
import java.net.URISyntaxException;
-import java.security.cert.X509Certificate;
import java.util.Collection;
import java.util.Enumeration;
import java.util.HashMap;
@@ -45,9 +44,8 @@ import org.apache.nifi.action.Operation;
import org.apache.nifi.cluster.context.ClusterContext;
import org.apache.nifi.cluster.context.ClusterContextThreadLocal;
import org.apache.nifi.cluster.manager.impl.WebClusterManager;
-import org.apache.nifi.web.security.DnUtils;
+import org.apache.nifi.web.security.ProxiedEntitiesUtils;
import org.apache.nifi.web.security.user.NiFiUserDetails;
-import org.apache.nifi.web.security.x509.X509CertificateExtractor;
import org.apache.nifi.util.NiFiProperties;
import org.apache.nifi.web.api.entity.Entity;
import org.apache.nifi.web.api.request.ClientIdParameter;
@@ -55,6 +53,8 @@ import org.apache.nifi.web.util.WebUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.builder.ReflectionToStringBuilder;
import org.apache.commons.lang3.builder.ToStringStyle;
+import org.apache.nifi.user.NiFiUser;
+import org.apache.nifi.web.security.user.NiFiUserUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.security.core.Authentication;
@@ -362,14 +362,12 @@ public abstract class ApplicationResource {
result.put(PROXY_SCHEME_HTTP_HEADER, httpServletRequest.getScheme());
}
- // if this is a secure request, add the custom headers for proxying user requests
- final X509Certificate cert = new X509CertificateExtractor().extractClientCertificate(httpServletRequest);
- if (cert != null) {
+ if (httpServletRequest.isSecure()) {
// add the certificate DN to the proxy chain
- final String xProxiedEntitiesChain = DnUtils.getXProxiedEntitiesChain(httpServletRequest);
- if (StringUtils.isNotBlank(xProxiedEntitiesChain)) {
- result.put(PROXIED_ENTITIES_CHAIN_HTTP_HEADER, xProxiedEntitiesChain);
+ final NiFiUser user = NiFiUserUtils.getNiFiUser();
+ if (user != null) {
+ result.put(PROXIED_ENTITIES_CHAIN_HTTP_HEADER, ProxiedEntitiesUtils.buildProxiedEntitiesChainString(user));
}
// add the user's authorities (if any) to the headers
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ControllerResource.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ControllerResource.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ControllerResource.java
index 4883721..6e6739d 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ControllerResource.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ControllerResource.java
@@ -79,6 +79,7 @@ import org.apache.nifi.web.api.request.IntegerParameter;
import org.apache.nifi.web.api.request.LongParameter;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.web.api.entity.ControllerServiceTypesEntity;
+import org.apache.nifi.web.api.entity.IdentityEntity;
import org.apache.nifi.web.api.entity.ReportingTaskTypesEntity;
import org.springframework.security.access.prepost.PreAuthorize;
@@ -239,7 +240,7 @@ public class ControllerResource extends ApplicationResource {
public ProcessGroupResource getGroupResource(
@ApiParam(
value = "The id of the process group that is the parent of the requested resource(s). If the desired process group is "
- + "the root group an alias 'root' may be used as the process-group-id.",
+ + "the root group an alias 'root' may be used as the process-group-id.",
required = true
)
@PathParam("process-group-id") String groupId) {
@@ -454,13 +455,13 @@ public class ControllerResource extends ApplicationResource {
@ApiOperation(
value = "Gets the current revision of this NiFi",
notes = "NiFi employs an optimistic locking strategy where the client must include a revision in their request when "
- + "performing an update. If the specified revision does not match the current base revision a 409 status code "
- + "is returned. The revision is comprised of a clientId and a version number. The version is a simple integer "
- + "value that is incremented with each change. Including the most recent version tells NiFi that your working "
- + "with the most recent flow. In addition to the version the client who is performing the updates is recorded. "
- + "This allows the same client to submit multiple requests without having to wait for the previously ones to "
- + "return. Invoking this endpoint will return the current base revision. It is also available when retrieving "
- + "a process group and in the response of all mutable requests.",
+ + "performing an update. If the specified revision does not match the current base revision a 409 status code "
+ + "is returned. The revision is comprised of a clientId and a version number. The version is a simple integer "
+ + "value that is incremented with each change. Including the most recent version tells NiFi that your working "
+ + "with the most recent flow. In addition to the version the client who is performing the updates is recorded. "
+ + "This allows the same client to submit multiple requests without having to wait for the previously ones to "
+ + "return. Invoking this endpoint will return the current base revision. It is also available when retrieving "
+ + "a process group and in the response of all mutable requests.",
response = Entity.class,
authorizations = {
@Authorization(value = "Read Only", type = "ROLE_MONITOR"),
@@ -845,6 +846,47 @@ public class ControllerResource extends ApplicationResource {
}
/**
+ * Retrieves the identity of the user making the request.
+ *
+ * @param clientId Optional client id. If the client id is not specified, a new one will be generated. This value (whether specified or generated) is included in the response.
+ * @return An identityEntity
+ */
+ @GET
+ @Consumes(MediaType.WILDCARD)
+ @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML})
+ @Path("/identity")
+ @ApiOperation(
+ value = "Retrieves the user identity of the user making the request",
+ response = IdentityEntity.class
+ )
+ public Response getIdentity(
+ @ApiParam(
+ value = "If the client id is not specified, new one will be generated. This value (whether specified or generated) is included in the response.",
+ required = false
+ )
+ @QueryParam(CLIENT_ID) @DefaultValue(StringUtils.EMPTY) ClientIdParameter clientId) {
+
+ // note that the cluster manager will handle this request directly
+ final NiFiUser user = NiFiUserUtils.getNiFiUser();
+ if (user == null) {
+ throw new WebApplicationException(new Throwable("Unable to access details for current user."));
+ }
+
+ // create the revision
+ final RevisionDTO revision = new RevisionDTO();
+ revision.setClientId(clientId.getClientId());
+
+ // create the response entity
+ IdentityEntity entity = new IdentityEntity();
+ entity.setRevision(revision);
+ entity.setUserId(user.getId());
+ entity.setIdentity(user.getUserName());
+
+ // generate the response
+ return clusterContext(generateOkResponse(entity)).build();
+ }
+
+ /**
* Retrieves the user details, including the authorities, about the user making the request.
*
* @param clientId Optional client id. If the client id is not specified, a new one will be generated. This value (whether specified or generated) is included in the response.
@@ -854,14 +896,17 @@ public class ControllerResource extends ApplicationResource {
@Consumes(MediaType.WILDCARD)
@Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML})
@Path("/authorities")
- @PreAuthorize("hasAnyRole('ROLE_MONITOR', 'ROLE_DFM', 'ROLE_ADMIN')")
+ @PreAuthorize("hasAnyRole('ROLE_MONITOR', 'ROLE_DFM', 'ROLE_ADMIN', 'ROLE_PROXY', 'ROLE_NIFI', 'ROLE_PROVENANCE')")
@ApiOperation(
value = "Retrieves the user details, including the authorities, about the user making the request",
response = AuthorityEntity.class,
authorizations = {
@Authorization(value = "Read Only", type = "ROLE_MONITOR"),
@Authorization(value = "Data Flow Manager", type = "ROLE_DFM"),
- @Authorization(value = "Administrator", type = "ROLE_ADMIN")
+ @Authorization(value = "Administrator", type = "ROLE_ADMIN"),
+ @Authorization(value = "Proxy", type = "ROLE_PROXY"),
+ @Authorization(value = "NiFi", type = "ROLE_NIFI"),
+ @Authorization(value = "Provenance", type = "ROLE_PROVENANCE")
}
)
@ApiResponses(
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/UserResource.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/UserResource.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/UserResource.java
index 4a61ef4..4b9d2ae 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/UserResource.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/UserResource.java
@@ -16,12 +16,14 @@
*/
package org.apache.nifi.web.api;
+import com.sun.jersey.api.Responses;
import com.wordnik.swagger.annotations.Api;
import com.wordnik.swagger.annotations.ApiOperation;
import com.wordnik.swagger.annotations.ApiParam;
import com.wordnik.swagger.annotations.ApiResponse;
import com.wordnik.swagger.annotations.ApiResponses;
import com.wordnik.swagger.annotations.Authorization;
+import java.net.URI;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -39,6 +41,7 @@ import javax.ws.rs.DefaultValue;
import javax.ws.rs.FormParam;
import javax.ws.rs.GET;
import javax.ws.rs.HttpMethod;
+import javax.ws.rs.POST;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
@@ -59,9 +62,11 @@ import org.apache.nifi.web.api.entity.UserSearchResultsEntity;
import org.apache.nifi.web.api.entity.UsersEntity;
import org.apache.nifi.web.api.request.ClientIdParameter;
import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.user.NiFiUser;
import org.apache.nifi.web.NiFiServiceFacade;
import static org.apache.nifi.web.api.ApplicationResource.CLIENT_ID;
import org.apache.nifi.web.api.dto.RevisionDTO;
+import org.apache.nifi.web.security.user.NiFiUserUtils;
import org.springframework.security.access.prepost.PreAuthorize;
/**
@@ -83,12 +88,35 @@ public class UserResource extends ApplicationResource {
private NiFiProperties properties;
private NiFiServiceFacade serviceFacade;
+ @POST
+ @Consumes(MediaType.WILDCARD)
+ @Produces(MediaType.TEXT_PLAIN)
+ @Path("") // necessary due to a bug in swagger
+ @ApiOperation(
+ value = "Creates a user",
+ response = String.class
+ )
+ public Response createUser() {
+ if (!properties.getSupportNewAccountRequests()) {
+ return Responses.notFound().entity("This NiFi does not support new account requests.").build();
+ }
+
+ final NiFiUser nifiUser = NiFiUserUtils.getNiFiUser();
+ if (nifiUser != null) {
+ throw new IllegalArgumentException("User account already created " + nifiUser.getIdentity());
+ }
+
+ // create an account request for the current user
+ final UserDTO user = serviceFacade.createUser();
+
+ final String uri = generateResourceUri("controller", "templates", user.getId());
+ return generateCreatedResponse(URI.create(uri), "Not authorized. User account created. Authorization pending.").build();
+ }
+
/**
* Gets all users that are registered within this Controller.
*
- * @param clientId Optional client id. If the client id is not specified, a
- * new one will be generated. This value (whether specified or generated) is
- * included in the response.
+ * @param clientId Optional client id. If the client id is not specified, a new one will be generated. This value (whether specified or generated) is included in the response.
* @param grouped Whether to return the users in their groups.
* @return A usersEntity.
*/
@@ -144,9 +172,7 @@ public class UserResource extends ApplicationResource {
/**
* Gets the details for the specified user.
*
- * @param clientId Optional client id. If the client id is not specified, a
- * new one will be generated. This value (whether specified or generated) is
- * included in the response.
+ * @param clientId Optional client id. If the client id is not specified, a new one will be generated. This value (whether specified or generated) is included in the response.
* @param id The user id.
* @return A userEntity.
*/
@@ -315,12 +341,9 @@ public class UserResource extends ApplicationResource {
* Updates the specified user.
*
* @param httpServletRequest request
- * @param clientId Optional client id. If the client id is not specified, a
- * new one will be generated. This value (whether specified or generated) is
- * included in the response.
+ * @param clientId Optional client id. If the client id is not specified, a new one will be generated. This value (whether specified or generated) is included in the response.
* @param id The id of the user to update.
- * @param rawAuthorities Array of authorities to assign to the specified
- * user.
+ * @param rawAuthorities Array of authorities to assign to the specified user.
* @param status The status of the specified users account.
* @param formParams form params
* @return A userEntity
@@ -491,9 +514,7 @@ public class UserResource extends ApplicationResource {
*
* @param httpServletRequest request
* @param id The user id
- * @param clientId Optional client id. If the client id is not specified, a
- * new one will be generated. This value (whether specified or generated) is
- * included in the response.
+ * @param clientId Optional client id. If the client id is not specified, a new one will be generated. This value (whether specified or generated) is included in the response.
* @return A userEntity.
*/
@DELETE
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/config/AccessDeniedExceptionMapper.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/config/AccessDeniedExceptionMapper.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/config/AccessDeniedExceptionMapper.java
index bc6dded..5d50e70 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/config/AccessDeniedExceptionMapper.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/config/AccessDeniedExceptionMapper.java
@@ -39,7 +39,7 @@ public class AccessDeniedExceptionMapper implements ExceptionMapper<AccessDenied
// get the current user
NiFiUser user = NiFiUserUtils.getNiFiUser();
if (user != null) {
- logger.info(String.format("%s does not have permission to access the requested resource. Returning %s response.", user.getDn(), Response.Status.FORBIDDEN));
+ logger.info(String.format("%s does not have permission to access the requested resource. Returning %s response.", user.getIdentity(), Response.Status.FORBIDDEN));
} else {
logger.info(String.format("User does not have permission to access the requested resource. Returning %s response.", Response.Status.FORBIDDEN));
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/config/InvalidAuthenticationExceptionMapper.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/config/InvalidAuthenticationExceptionMapper.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/config/InvalidAuthenticationExceptionMapper.java
new file mode 100644
index 0000000..14d5139
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/config/InvalidAuthenticationExceptionMapper.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.api.config;
+
+import javax.ws.rs.core.Response;
+import javax.ws.rs.ext.ExceptionMapper;
+import javax.ws.rs.ext.Provider;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.web.security.InvalidAuthenticationException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Maps access denied exceptions into a client response.
+ */
+@Provider
+public class InvalidAuthenticationExceptionMapper implements ExceptionMapper<InvalidAuthenticationException> {
+
+ private static final Logger logger = LoggerFactory.getLogger(InvalidAuthenticationExceptionMapper.class);
+
+ @Override
+ public Response toResponse(InvalidAuthenticationException exception) {
+ if (logger.isDebugEnabled()) {
+ logger.debug(StringUtils.EMPTY, exception);
+ }
+
+ return Response.status(Response.Status.UNAUTHORIZED).entity(exception.getMessage()).type("text/plain").build();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
index 87277ae..c16653e 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
@@ -2382,7 +2382,7 @@ public final class DtoFactory {
// create the user
UserDTO userDTO = new UserDTO();
userDTO.setId(String.valueOf(user.getId()));
- userDTO.setDn(user.getDn());
+ userDTO.setDn(user.getIdentity());
userDTO.setUserName(user.getUserName());
userDTO.setUserGroup(user.getUserGroup());
userDTO.setJustification(user.getJustification());
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java
index 0e3bcac..47ebd30 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java
@@ -113,6 +113,7 @@ import org.apache.nifi.web.api.dto.search.SearchResultsDTO;
import org.apache.nifi.web.api.dto.status.ControllerStatusDTO;
import org.apache.nifi.web.api.dto.status.ProcessGroupStatusDTO;
import org.apache.nifi.web.api.dto.status.StatusHistoryDTO;
+import org.apache.nifi.web.security.ProxiedEntitiesUtils;
import org.apache.nifi.web.security.user.NiFiUserUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -822,17 +823,7 @@ public class ControllerFacade {
final Map<String, String> attributes = event.getAttributes();
// calculate the dn chain
- final List<String> dnChain = new ArrayList<>();
-
- // build the dn chain
- NiFiUser chainedUser = user;
- do {
- // add the entry for this user
- dnChain.add(chainedUser.getDn());
-
- // go to the next user in the chain
- chainedUser = chainedUser.getChain();
- } while (chainedUser != null);
+ final List<String> dnChain = ProxiedEntitiesUtils.buildProxiedEntitiesChain(user);
// ensure the users in this chain are allowed to download this content
final DownloadAuthorization downloadAuthorization = userService.authorizeDownload(dnChain, attributes);
@@ -850,7 +841,7 @@ public class ControllerFacade {
final String type = event.getAttributes().get(CoreAttributes.MIME_TYPE.key());
// get the content
- final InputStream content = flowController.getContent(event, contentDirection, user.getDn(), uri);
+ final InputStream content = flowController.getContent(event, contentDirection, user.getIdentity(), uri);
return new DownloadableContent(filename, type, content);
} catch (final ContentNotFoundException cnfe) {
throw new ResourceNotFoundException("Unable to find the specified content.");
@@ -880,7 +871,7 @@ public class ControllerFacade {
}
// replay the flow file
- final ProvenanceEventRecord event = flowController.replayFlowFile(originalEvent, user.getDn());
+ final ProvenanceEventRecord event = flowController.replayFlowFile(originalEvent, user.getIdentity());
// convert the event record
return createProvenanceEventDto(event);
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java
index df271a5..565e5af 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java
@@ -321,7 +321,7 @@ public class StandardConnectionDAO extends ComponentDAO implements ConnectionDAO
throw new WebApplicationException(new Throwable("Unable to access details for current user."));
}
- return queue.dropFlowFiles(dropRequestId, user.getDn());
+ return queue.dropFlowFiles(dropRequestId, user.getIdentity());
}
@Override
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/filter/RequestLogger.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/filter/RequestLogger.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/filter/RequestLogger.java
index bfd2df6..9f63611 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/filter/RequestLogger.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/filter/RequestLogger.java
@@ -52,13 +52,13 @@ public class RequestLogger implements Filter {
final NiFiUser user = NiFiUserUtils.getNiFiUser();
// get the user details for the log message
- String dn = "<no user found>";
+ String identity = "<no user found>";
if (user != null) {
- dn = user.getDn();
+ identity = user.getIdentity();
}
// log the request attempt - response details will be logged later
- logger.info(String.format("Attempting request for (%s) %s %s (source ip: %s)", dn, request.getMethod(),
+ logger.info(String.format("Attempting request for (%s) %s %s (source ip: %s)", identity, request.getMethod(),
request.getRequestURL().toString(), request.getRemoteAddr()));
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/resources/nifi-web-api-context.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/resources/nifi-web-api-context.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/resources/nifi-web-api-context.xml
index e034baa..9f3d2f5 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/resources/nifi-web-api-context.xml
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/resources/nifi-web-api-context.xml
@@ -241,12 +241,21 @@
<bean id="systemDiagnosticsResource" class="org.apache.nifi.web.api.SystemDiagnosticsResource" scope="singleton">
<property name="serviceFacade" ref="serviceFacade"/>
</bean>
+ <bean id="accessResource" class="org.apache.nifi.web.api.AccessResource" scope="singleton">
+ <property name="properties" ref="nifiProperties"/>
+ <property name="certificateExtractor" ref="certificateExtractor"/>
+ <property name="certificateIdentityProvider" ref="certificateIdentityProvider"/>
+ <property name="loginIdentityProvider" ref="loginIdentityProvider"/>
+ <property name="jwtService" ref="jwtService"/>
+ <property name="userDetailsService" ref="userDetailsService"/>
+ </bean>
<!-- configuration for jaxb serialization -->
<bean class="org.apache.nifi.web.util.ObjectMapperResolver" scope="singleton"/>
<!-- exception mapping -->
<bean class="org.apache.nifi.web.api.config.AccessDeniedExceptionMapper" scope="singleton"/>
+ <bean class="org.apache.nifi.web.api.config.InvalidAuthenticationExceptionMapper" scope="singleton"/>
<bean class="org.apache.nifi.web.api.config.AuthenticationCredentialsNotFoundExceptionMapper" scope="singleton"/>
<bean class="org.apache.nifi.web.api.config.AccountNotFoundExceptionMapper" scope="singleton"/>
<bean class="org.apache.nifi.web.api.config.AdministrationExceptionMapper" scope="singleton"/>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/webapp/WEB-INF/web.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/webapp/WEB-INF/web.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/webapp/WEB-INF/web.xml
index 4ce319e..b57998d 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/webapp/WEB-INF/web.xml
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/webapp/WEB-INF/web.xml
@@ -16,15 +16,12 @@
<web-app version="3.0" xmlns="http://java.sun.com/xml/ns/javaee" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_3_0.xsd">
<display-name>nifi-api</display-name>
<context-param>
+ <param-name>contextClass</param-name>
+ <param-value>org.springframework.web.context.support.AnnotationConfigWebApplicationContext</param-value>
+ </context-param>
+ <context-param>
<param-name>contextConfigLocation</param-name>
- <param-value>
- classpath:nifi-context.xml
- classpath:nifi-web-api-context.xml
- classpath:nifi-web-security-context.xml
- classpath:nifi-administration-context.xml
- classpath:nifi-cluster-manager-context.xml
- classpath:nifi-cluster-protocol-context.xml
- </param-value>
+ <param-value>org.apache.nifi.web</param-value>
</context-param>
<listener>
<listener-class>org.springframework.web.context.ContextLoaderListener</listener-class>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/integration/accesscontrol/AccessTokenEndpointTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/integration/accesscontrol/AccessTokenEndpointTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/integration/accesscontrol/AccessTokenEndpointTest.java
new file mode 100644
index 0000000..82fe73a
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/integration/accesscontrol/AccessTokenEndpointTest.java
@@ -0,0 +1,292 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.integration.accesscontrol;
+
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientResponse;
+import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
+import javax.net.ssl.SSLContext;
+import org.apache.commons.io.FileUtils;
+import org.apache.nifi.integration.util.NiFiTestServer;
+import org.apache.nifi.integration.util.NiFiTestUser;
+import org.apache.nifi.integration.util.SourceTestProcessor;
+import org.apache.nifi.nar.ExtensionManager;
+import org.apache.nifi.nar.NarClassLoaders;
+import org.apache.nifi.security.util.SslContextFactory;
+import org.apache.nifi.util.NiFiProperties;
+import org.apache.nifi.web.api.dto.AccessConfigurationDTO;
+import org.apache.nifi.web.api.dto.AccessStatusDTO;
+import org.apache.nifi.web.api.dto.ProcessorDTO;
+import org.apache.nifi.web.api.dto.RevisionDTO;
+import org.apache.nifi.web.api.entity.AccessConfigurationEntity;
+import org.apache.nifi.web.api.entity.AccessStatusEntity;
+import org.apache.nifi.web.api.entity.ProcessorEntity;
+import org.apache.nifi.web.util.WebUtils;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Access token endpoint test.
+ */
+public class AccessTokenEndpointTest {
+
+ private static final String CLIENT_ID = "token-endpoint-id";
+ private static final String CONTEXT_PATH = "/nifi-api";
+ private static final String FLOW_XML_PATH = "target/test-classes/access-control/flow-admin.xml";
+
+ private static NiFiTestServer SERVER;
+ private static NiFiTestUser TOKEN_USER;
+ private static String BASE_URL;
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ // configure the location of the nifi properties
+ File nifiPropertiesFile = new File("src/test/resources/access-control/nifi.properties");
+ System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, nifiPropertiesFile.getAbsolutePath());
+
+ // update the flow.xml property
+ NiFiProperties props = NiFiProperties.getInstance();
+ props.setProperty("nifi.flow.configuration.file", FLOW_XML_PATH);
+
+ // delete the database directory to avoid issues with re-registration in testRequestAccessUsingToken
+ FileUtils.deleteDirectory(props.getDatabaseRepositoryPath().toFile());
+
+ // load extensions
+ NarClassLoaders.load(props);
+ ExtensionManager.discoverExtensions();
+
+ // start the server
+ SERVER = new NiFiTestServer("src/main/webapp", CONTEXT_PATH);
+ SERVER.startServer();
+ SERVER.loadFlow();
+
+ // get the base url
+ BASE_URL = SERVER.getBaseUrl() + CONTEXT_PATH;
+
+ // create the user
+ final Client client = WebUtils.createClient(null, createTrustContext(props));
+ TOKEN_USER = new NiFiTestUser(client, null);
+ }
+
+ private static SSLContext createTrustContext(final NiFiProperties props) throws Exception {
+ return SslContextFactory.createTrustSslContext(props.getProperty(NiFiProperties.SECURITY_TRUSTSTORE),
+ props.getProperty(NiFiProperties.SECURITY_TRUSTSTORE_PASSWD).toCharArray(),
+ props.getProperty(NiFiProperties.SECURITY_TRUSTSTORE_TYPE), "TLS");
+ }
+
+ // -----------
+ // LOGIN CONIG
+ // -----------
+ /**
+ * Test getting access configuration.
+ *
+ * @throws Exception ex
+ */
+ @Test
+ public void testGetAccessConfig() throws Exception {
+ String url = BASE_URL + "/access/config";
+
+ ClientResponse response = TOKEN_USER.testGet(url);
+
+ // ensure the request is successful
+ Assert.assertEquals(200, response.getStatus());
+
+ // extract the process group
+ AccessConfigurationEntity accessConfigEntity = response.getEntity(AccessConfigurationEntity.class);
+
+ // ensure there is content
+ Assert.assertNotNull(accessConfigEntity);
+
+ // extract the process group dto
+ AccessConfigurationDTO accessConfig = accessConfigEntity.getConfig();
+
+ // verify config
+ Assert.assertTrue(accessConfig.getSupportsLogin());
+ Assert.assertFalse(accessConfig.getSupportsAnonymous());
+ }
+
+ /**
+ * Obtains a token and creates a processor using it.
+ *
+ * @throws Exception ex
+ */
+ @Test
+ public void testCreateProcessorUsingToken() throws Exception {
+ String url = BASE_URL + "/access/token";
+
+ ClientResponse response = TOKEN_USER.testCreateToken(url, "user@nifi", "whateve");
+
+ // ensure the request is successful
+ Assert.assertEquals(201, response.getStatus());
+
+ // get the token
+ String token = response.getEntity(String.class);
+
+ // attempt to create a processor with it
+ createProcessor(token);
+ }
+
+ private ProcessorDTO createProcessor(final String token) throws Exception {
+ String url = BASE_URL + "/controller/process-groups/root/processors";
+
+ // authorization header
+ Map<String, String> headers = new HashMap<>();
+ headers.put("Authorization", "Bearer " + token);
+
+ // create the processor
+ ProcessorDTO processor = new ProcessorDTO();
+ processor.setName("Copy");
+ processor.setType(SourceTestProcessor.class.getName());
+
+ // create the revision
+ final RevisionDTO revision = new RevisionDTO();
+ revision.setClientId(CLIENT_ID);
+ revision.setVersion(NiFiTestUser.REVISION);
+
+ // create the entity body
+ ProcessorEntity entity = new ProcessorEntity();
+ entity.setRevision(revision);
+ entity.setProcessor(processor);
+
+ // perform the request
+ ClientResponse response = TOKEN_USER.testPostWithHeaders(url, entity, headers);
+
+ // ensure the request is successful
+ Assert.assertEquals(201, response.getStatus());
+
+ // get the entity body
+ entity = response.getEntity(ProcessorEntity.class);
+
+ // verify creation
+ processor = entity.getProcessor();
+ Assert.assertEquals("Copy", processor.getName());
+ Assert.assertEquals("org.apache.nifi.integration.util.SourceTestProcessor", processor.getType());
+
+ return processor;
+ }
+
+ /**
+ * Verifies the response when bad credentials are specified.
+ *
+ * @throws Exception ex
+ */
+ @Test
+ public void testInvalidCredentials() throws Exception {
+ String url = BASE_URL + "/access/token";
+
+ ClientResponse response = TOKEN_USER.testCreateToken(url, "user@nifi", "not a real password");
+
+ // ensure the request is successful
+ Assert.assertEquals(400, response.getStatus());
+ }
+
+ /**
+ * Verifies the response when the user is known.
+ *
+ * @throws Exception ex
+ */
+ @Test
+ public void testUnkownUser() throws Exception {
+ String url = BASE_URL + "/access/token";
+
+ ClientResponse response = TOKEN_USER.testCreateToken(url, "not a real user", "not a real password");
+
+ // ensure the request is successful
+ Assert.assertEquals(400, response.getStatus());
+ }
+
+ /**
+ * Request access using access token.
+ *
+ * @throws Exception ex
+ */
+ @Test
+ public void testRequestAccessUsingToken() throws Exception {
+ String accessStatusUrl = BASE_URL + "/access";
+ String accessTokenUrl = BASE_URL + "/access/token";
+ String registrationUrl = BASE_URL + "/controller/users";
+
+ ClientResponse response = TOKEN_USER.testGet(accessStatusUrl);
+
+ // ensure the request is successful
+ Assert.assertEquals(200, response.getStatus());
+
+ AccessStatusEntity accessStatusEntity = response.getEntity(AccessStatusEntity.class);
+ AccessStatusDTO accessStatus = accessStatusEntity.getAccessStatus();
+
+ // verify unknown
+ Assert.assertEquals("UNKNOWN", accessStatus.getStatus());
+
+ response = TOKEN_USER.testCreateToken(accessTokenUrl, "unregistered-user@nifi", "password");
+
+ // ensure the request is successful
+ Assert.assertEquals(201, response.getStatus());
+
+ // get the token
+ String token = response.getEntity(String.class);
+
+ // authorization header
+ Map<String, String> headers = new HashMap<>();
+ headers.put("Authorization", "Bearer " + token);
+
+ // check the status with the token
+ response = TOKEN_USER.testGetWithHeaders(accessStatusUrl, null, headers);
+
+ // ensure the request is successful
+ Assert.assertEquals(200, response.getStatus());
+
+ accessStatusEntity = response.getEntity(AccessStatusEntity.class);
+ accessStatus = accessStatusEntity.getAccessStatus();
+
+ // verify unregistered
+ Assert.assertEquals("UNREGISTERED", accessStatus.getStatus());
+
+ response = TOKEN_USER.testRegisterUser(registrationUrl, "Gimme access", headers);
+
+ // ensure the request is successful
+ Assert.assertEquals(201, response.getStatus());
+
+ // check the status with the token
+ response = TOKEN_USER.testGetWithHeaders(accessStatusUrl, null, headers);
+
+ // ensure the request is successful
+ Assert.assertEquals(200, response.getStatus());
+
+ accessStatusEntity = response.getEntity(AccessStatusEntity.class);
+ accessStatus = accessStatusEntity.getAccessStatus();
+
+ // verify unregistered
+ Assert.assertEquals("NOT_ACTIVE", accessStatus.getStatus());
+ }
+
+ @AfterClass
+ public static void cleanup() throws Exception {
+ // shutdown the server
+ SERVER.shutdownServer();
+ SERVER = null;
+
+ // look for the flow.xml
+ File flow = new File(FLOW_XML_PATH);
+ if (flow.exists()) {
+ flow.delete();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/integration/util/NiFiTestAuthorizationProvider.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/integration/util/NiFiTestAuthorizationProvider.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/integration/util/NiFiTestAuthorizationProvider.java
index d51b7df..d29be92 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/integration/util/NiFiTestAuthorizationProvider.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/integration/util/NiFiTestAuthorizationProvider.java
@@ -37,7 +37,7 @@ import org.apache.nifi.authorization.DownloadAuthorization;
*/
public class NiFiTestAuthorizationProvider implements AuthorityProvider {
- private Map<String, Set<Authority>> users;
+ private final Map<String, Set<Authority>> users;
/**
* Creates a new FileAuthorizationProvider.
@@ -48,6 +48,7 @@ public class NiFiTestAuthorizationProvider implements AuthorityProvider {
users.put("CN=Lastname Firstname Middlename monitor, OU=Unknown, OU=Unknown, OU=Unknown, O=Unknown, C=Unknown", EnumSet.of(Authority.ROLE_MONITOR));
users.put("CN=Lastname Firstname Middlename dfm, OU=Unknown, OU=Unknown, OU=Unknown, O=Unknown, C=Unknown", EnumSet.of(Authority.ROLE_DFM));
users.put("CN=Lastname Firstname Middlename admin, OU=Unknown, OU=Unknown, OU=Unknown, O=Unknown, C=Unknown", EnumSet.of(Authority.ROLE_ADMIN));
+ users.put("user@nifi", EnumSet.of(Authority.ROLE_DFM));
}
@Override
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/integration/util/NiFiTestLoginIdentityProvider.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/integration/util/NiFiTestLoginIdentityProvider.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/integration/util/NiFiTestLoginIdentityProvider.java
new file mode 100644
index 0000000..c023ce1
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/integration/util/NiFiTestLoginIdentityProvider.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.integration.util;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import org.apache.nifi.authorization.exception.ProviderCreationException;
+import org.apache.nifi.authentication.AuthenticationResponse;
+import org.apache.nifi.authentication.LoginCredentials;
+import org.apache.nifi.authentication.LoginIdentityProvider;
+import org.apache.nifi.authentication.LoginIdentityProviderConfigurationContext;
+import org.apache.nifi.authentication.LoginIdentityProviderInitializationContext;
+import org.apache.nifi.authentication.exception.IdentityAccessException;
+import org.apache.nifi.authentication.exception.InvalidLoginCredentialsException;
+
+/**
+ *
+ */
+public class NiFiTestLoginIdentityProvider implements LoginIdentityProvider {
+
+ private final Map<String, String> users;
+
+ /**
+ * Creates a new FileAuthorizationProvider.
+ */
+ public NiFiTestLoginIdentityProvider() {
+ users = new HashMap<>();
+ users.put("user@nifi", "whateve");
+ users.put("unregistered-user@nifi", "password");
+ }
+
+ private void checkUser(final String user, final String password) {
+ if (!users.containsKey(user)) {
+ throw new InvalidLoginCredentialsException("Unknown user");
+ }
+
+ if (!users.get(user).equals(password)) {
+ throw new InvalidLoginCredentialsException("Invalid password");
+ }
+ }
+
+ @Override
+ public AuthenticationResponse authenticate(LoginCredentials credentials) throws InvalidLoginCredentialsException, IdentityAccessException {
+ checkUser(credentials.getUsername(), credentials.getPassword());
+ return new AuthenticationResponse(credentials.getUsername(), credentials.getUsername(), TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS), getClass().getSimpleName());
+ }
+
+ @Override
+ public void initialize(LoginIdentityProviderInitializationContext initializationContext) throws ProviderCreationException {
+ }
+
+ @Override
+ public void onConfigured(LoginIdentityProviderConfigurationContext configurationContext) throws ProviderCreationException {
+ }
+
+ @Override
+ public void preDestruction() {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/integration/util/NiFiTestServer.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/integration/util/NiFiTestServer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/integration/util/NiFiTestServer.java
index 42b0aab..38c2d41 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/integration/util/NiFiTestServer.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/integration/util/NiFiTestServer.java
@@ -78,8 +78,12 @@ public class NiFiTestServer {
private void createSecureConnector() {
org.eclipse.jetty.util.ssl.SslContextFactory contextFactory = new org.eclipse.jetty.util.ssl.SslContextFactory();
- // need client auth
- contextFactory.setNeedClientAuth(properties.getNeedClientAuth());
+ // require client auth when not supporting login or anonymous access
+ if (StringUtils.isBlank(properties.getProperty(NiFiProperties.SECURITY_USER_LOGIN_IDENTITY_PROVIDER)) && properties.getAnonymousAuthorities().isEmpty()) {
+ contextFactory.setNeedClientAuth(true);
+ } else {
+ contextFactory.setWantClientAuth(true);
+ }
/* below code sets JSSE system properties when values are provided */
// keystore properties
@@ -163,7 +167,6 @@ public class NiFiTestServer {
}
public Client getClient() {
- // create the client
return WebUtils.createClient(null, SslContextFactory.createSslContext(properties));
}
[50/50] [abbrv] nifi git commit: Merge branch 'NIFI-655' of
https://git-wip-us.apache.org/repos/asf/nifi into NIFI-655
Posted by mc...@apache.org.
Merge branch 'NIFI-655' of https://git-wip-us.apache.org/repos/asf/nifi into NIFI-655
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/769f19ee
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/769f19ee
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/769f19ee
Branch: refs/heads/NIFI-655
Commit: 769f19ee867f69e2d3bf157cae0a1f09ea4f24fa
Parents: aaf14c4 5ef53b6
Author: Matt Gilman <ma...@gmail.com>
Authored: Mon Nov 23 15:21:47 2015 -0500
Committer: Matt Gilman <ma...@gmail.com>
Committed: Mon Nov 23 15:21:47 2015 -0500
----------------------------------------------------------------------
----------------------------------------------------------------------
[06/50] [abbrv] nifi git commit: NIFI-973: Updated main Help page to
provide a link to the Getting Started Guide.
Posted by mc...@apache.org.
NIFI-973: Updated main Help page to provide a link to the Getting Started Guide.
Reviewed by Aldrin Piri (aldrin@apache.org)
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/da9f40b3
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/da9f40b3
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/da9f40b3
Branch: refs/heads/NIFI-655
Commit: da9f40b3f0aa29ba5e8ba4e257e953b52c6493e6
Parents: 528dab7
Author: Mark Payne <ma...@hotmail.com>
Authored: Sun Nov 15 08:38:51 2015 -0500
Committer: Tony Kurc <tr...@gmail.com>
Committed: Sun Nov 15 08:38:51 2015 -0500
----------------------------------------------------------------------
.../nifi-web-docs/src/main/webapp/WEB-INF/jsp/documentation.jsp | 1 +
.../nifi-web/nifi-web-docs/src/main/webapp/js/application.js | 2 +-
2 files changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/da9f40b3/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-docs/src/main/webapp/WEB-INF/jsp/documentation.jsp
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-docs/src/main/webapp/WEB-INF/jsp/documentation.jsp b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-docs/src/main/webapp/WEB-INF/jsp/documentation.jsp
index 9690def..31ed963 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-docs/src/main/webapp/WEB-INF/jsp/documentation.jsp
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-docs/src/main/webapp/WEB-INF/jsp/documentation.jsp
@@ -43,6 +43,7 @@
<div id="general-links" class="component-links">
<ul>
<li class="component-item"><a class="component-link overview" href="html/overview.html" target="component-usage">Overview</a></li>
+ <li class="component-item"><a class="component-link getting-started" href="html/getting-started.html" target="component-usage">Getting Started</a></li>
<li class="component-item"><a class="component-link user-guide" href="html/user-guide.html" target="component-usage">User Guide</a></li>
<li class="component-item"><a class="component-link expression-language-guide" href="html/expression-language-guide.html" target="component-usage">Expression Language Guide</a></li>
<li class="component-item"><a class="component-link admin-guide" href="html/administration-guide.html" target="component-usage">Admin Guide</a></li>
http://git-wip-us.apache.org/repos/asf/nifi/blob/da9f40b3/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-docs/src/main/webapp/js/application.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-docs/src/main/webapp/js/application.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-docs/src/main/webapp/js/application.js
index 1261810..cdc062e 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-docs/src/main/webapp/js/application.js
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-docs/src/main/webapp/js/application.js
@@ -262,7 +262,7 @@ $(document).ready(function () {
});
// listen for on the rest api and user guide and developer guide and admin guide and overview
- $('a.rest-api, a.user-guide, a.developer-guide, a.admin-guide, a.overview, a.expression-language-guide').on('click', function() {
+ $('a.rest-api, a.user-guide, a.developer-guide, a.admin-guide, a.overview, a.expression-language-guide, a.getting-started').on('click', function() {
selectComponent($(this).text());
});
[20/50] [abbrv] nifi git commit: NIFI-1168: Ensure that processors
with only looping connections are scheduled to run, even if the connections
have no FlowFiles; expose these details to processor developers;
update documentation
Posted by mc...@apache.org.
NIFI-1168: Ensure that processors with only looping
connections are scheduled to run, even if the connections have no FlowFiles;
expose these details to processor developers; update documentation
Signed-off-by: Aldrin Piri <al...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/69bce2c2
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/69bce2c2
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/69bce2c2
Branch: refs/heads/NIFI-655
Commit: 69bce2c2db2619e0323434228c31f98630e43372
Parents: 773576e
Author: Mark Payne <ma...@hotmail.com>
Authored: Wed Nov 18 14:53:30 2015 -0500
Committer: Aldrin Piri <al...@apache.org>
Committed: Wed Nov 18 14:53:30 2015 -0500
----------------------------------------------------------------------
.../annotation/behavior/TriggerWhenEmpty.java | 26 +++++--
.../apache/nifi/processor/ProcessContext.java | 10 ++-
.../apache/nifi/util/MockProcessContext.java | 12 ++-
.../nifi/util/StandardProcessorTestRunner.java | 5 ++
.../java/org/apache/nifi/util/TestRunner.java | 10 ++-
.../documentation/mock/MockProcessContext.java | 5 ++
.../scheduling/ConnectableProcessContext.java | 6 ++
.../tasks/ContinuallyRunProcessorTask.java | 46 +++++++----
.../nifi/processor/StandardProcessContext.java | 6 ++
.../processor/StandardSchedulingContext.java | 5 ++
.../java/org/apache/nifi/util/Connectables.java | 12 +++
.../tasks/TestContinuallyRunProcessorTask.java | 82 ++++++++++++++++++++
.../nifi/processors/standard/ExecuteSQL.java | 6 +-
13 files changed, 207 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/69bce2c2/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/TriggerWhenEmpty.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/TriggerWhenEmpty.java b/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/TriggerWhenEmpty.java
index f442b80..6fac03b 100644
--- a/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/TriggerWhenEmpty.java
+++ b/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/TriggerWhenEmpty.java
@@ -24,14 +24,30 @@ import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/**
+ * <p>
* Marker annotation a {@link org.apache.nifi.processor.Processor Processor}
* implementation can use to indicate that the Processor should still be
- * triggered even when it has no data in its work queue. By default, Processors
- * which have no non-self incoming edges will be triggered even if there is no
- * work in its queue. However, Processors that have non-self incoming edges will
- * only be triggered if they have work in their queue or they present this
- * annotation.
+ * triggered even when it has no data in its work queue.
+ * </p>
*
+ * <p>
+ * A Processor is scheduled to be triggered based on its configured Scheduling Period
+ * and Scheduling Strategy. However, when the scheduling period elapses, the Processor
+ * will not be scheduled if it has no work to do. Normally, a Processor is said to have
+ * work to do if one of the following circumstances is true:
+ * </p>
+ *
+ * <ul>
+ * <li>An incoming Connection has data in its queue</li>
+ * <li>The Processor has no incoming Connections.</li>
+ * <li>All incoming Connections are self-loops (both the source and destination of the Connection are the same Processor).
+ * </ul>
+ *
+ * <p>
+ * If the Processor needs to be triggered to run even when the above conditions are all
+ * <code>false</code>, the Processor's class can be annotated with this annotation, which
+ * will cause the Processor to be triggered, even if its incoming queues are empty.
+ * </p>
*/
@Documented
@Target({ElementType.TYPE})
http://git-wip-us.apache.org/repos/asf/nifi/blob/69bce2c2/nifi-api/src/main/java/org/apache/nifi/processor/ProcessContext.java
----------------------------------------------------------------------
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/ProcessContext.java b/nifi-api/src/main/java/org/apache/nifi/processor/ProcessContext.java
index bb69c65..7488b2d 100644
--- a/nifi-api/src/main/java/org/apache/nifi/processor/ProcessContext.java
+++ b/nifi-api/src/main/java/org/apache/nifi/processor/ProcessContext.java
@@ -133,9 +133,17 @@ public interface ProcessContext {
boolean hasIncomingConnection();
/**
+ * @return <code>true</code> if the processor has one or more incoming connections for
+ * which the source of the connection is NOT the processor; returns <code>false</code> if
+ * the processor has no incoming connections or if all incoming connections are self-loops
+ * (i.e., the processor is also the source of all incoming connections).
+ */
+ boolean hasNonLoopConnection();
+
+ /**
* @param relationship a relationship to check for connections
* @return true if the relationship has one or more outbound connections,
- * false otherwise
+ * false otherwise
*/
boolean hasConnection(Relationship relationship);
http://git-wip-us.apache.org/repos/asf/nifi/blob/69bce2c2/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessContext.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessContext.java b/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessContext.java
index a350ecb..49021d1 100644
--- a/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessContext.java
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessContext.java
@@ -48,7 +48,8 @@ public class MockProcessContext extends MockControllerServiceLookup implements S
private boolean yieldCalled = false;
private boolean enableExpressionValidation = false;
private boolean allowExpressionValidation = true;
- private boolean incomingConnection = true;
+ private volatile boolean incomingConnection = true;
+ private volatile boolean nonLoopConnection = true;
private volatile Set<Relationship> connections = new HashSet<>();
private volatile Set<Relationship> unavailableRelationships = new HashSet<>();
@@ -305,6 +306,15 @@ public class MockProcessContext extends MockControllerServiceLookup implements S
return this.connections.contains(relationship);
}
+ public void setNonLoopConnection(final boolean hasNonLoopConnection) {
+ this.nonLoopConnection = hasNonLoopConnection;
+ }
+
+ @Override
+ public boolean hasNonLoopConnection() {
+ return nonLoopConnection;
+ }
+
public void addConnection(final Relationship relationship) {
this.connections.add(relationship);
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/69bce2c2/nifi-mock/src/main/java/org/apache/nifi/util/StandardProcessorTestRunner.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/StandardProcessorTestRunner.java b/nifi-mock/src/main/java/org/apache/nifi/util/StandardProcessorTestRunner.java
index 0d00cc8..2f384ba 100644
--- a/nifi-mock/src/main/java/org/apache/nifi/util/StandardProcessorTestRunner.java
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/StandardProcessorTestRunner.java
@@ -524,6 +524,11 @@ public class StandardProcessorTestRunner implements TestRunner {
}
@Override
+ public void setNonLoopConnection(final boolean hasNonLoopConnection) {
+ context.setNonLoopConnection(hasNonLoopConnection);
+ }
+
+ @Override
public void addConnection(Relationship relationship) {
context.addConnection(relationship);
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/69bce2c2/nifi-mock/src/main/java/org/apache/nifi/util/TestRunner.java
----------------------------------------------------------------------
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/TestRunner.java b/nifi-mock/src/main/java/org/apache/nifi/util/TestRunner.java
index ec901fe..b1e7c8c 100644
--- a/nifi-mock/src/main/java/org/apache/nifi/util/TestRunner.java
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/TestRunner.java
@@ -492,11 +492,19 @@ public interface TestRunner {
* Indicates to the framework that the configured processor has one or more
* incoming connections.
*
- * @param hasIncomingConnection whether or not the configured processor has an incoming connection
+ * @param hasIncomingConnection whether or not the configured processor should behave as though it has an incoming connection
*/
void setIncomingConnection(boolean hasIncomingConnection);
/**
+ * Indicates to the framework that the configured processor has one or more incoming
+ * connections for which the processor is not also the source.
+ *
+ * @param hasNonLoopConnection whether or not the configured processor should behave as though it has a non-looping incoming connection
+ */
+ void setNonLoopConnection(boolean hasNonLoopConnection);
+
+ /**
* Indicates to the Framework that the configured processor has a connection for the given Relationship.
*
* @param relationship that has a connection
http://git-wip-us.apache.org/repos/asf/nifi/blob/69bce2c2/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/main/java/org/apache/nifi/documentation/mock/MockProcessContext.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/main/java/org/apache/nifi/documentation/mock/MockProcessContext.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/main/java/org/apache/nifi/documentation/mock/MockProcessContext.java
index 3238d4a..38c9fc9 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/main/java/org/apache/nifi/documentation/mock/MockProcessContext.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/main/java/org/apache/nifi/documentation/mock/MockProcessContext.java
@@ -89,6 +89,11 @@ public class MockProcessContext implements ProcessContext {
}
@Override
+ public boolean hasNonLoopConnection() {
+ return true;
+ }
+
+ @Override
public boolean hasConnection(Relationship relationship) {
return false;
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/69bce2c2/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/ConnectableProcessContext.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/ConnectableProcessContext.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/ConnectableProcessContext.java
index da7162e..08e2504 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/ConnectableProcessContext.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/ConnectableProcessContext.java
@@ -37,6 +37,7 @@ import org.apache.nifi.processor.DataUnit;
import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.util.Connectables;
/**
* This class is essentially an empty shell for {@link Connectable}s that are not Processors
@@ -197,6 +198,11 @@ public class ConnectableProcessContext implements ProcessContext {
}
@Override
+ public boolean hasNonLoopConnection() {
+ return Connectables.hasNonLoopConnection(connectable);
+ }
+
+ @Override
public boolean hasConnection(Relationship relationship) {
Set<Connection> connections = connectable.getConnections(relationship);
return connections != null && !connections.isEmpty();
http://git-wip-us.apache.org/repos/asf/nifi/blob/69bce2c2/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/tasks/ContinuallyRunProcessorTask.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/tasks/ContinuallyRunProcessorTask.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/tasks/ContinuallyRunProcessorTask.java
index d4d595b..dd12824 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/tasks/ContinuallyRunProcessorTask.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/tasks/ContinuallyRunProcessorTask.java
@@ -72,30 +72,45 @@ public class ContinuallyRunProcessorTask implements Callable<Boolean> {
this.processContext = processContext;
}
+ static boolean isRunOnCluster(final ProcessorNode procNode, final boolean isClustered, final boolean isPrimary) {
+ return !procNode.isIsolated() || !isClustered || isPrimary;
+ }
+
+ static boolean isYielded(final ProcessorNode procNode) {
+ return procNode.getYieldExpiration() >= System.currentTimeMillis();
+ }
+
+ static boolean isWorkToDo(final ProcessorNode procNode) {
+ return procNode.isTriggerWhenEmpty() || !procNode.hasIncomingConnection() || !Connectables.hasNonLoopConnection(procNode) || Connectables.flowFilesQueued(procNode);
+ }
+
@Override
@SuppressWarnings("deprecation")
public Boolean call() {
// make sure processor is not yielded
- boolean shouldRun = (procNode.getYieldExpiration() < System.currentTimeMillis());
- if (!shouldRun) {
+ if (isYielded(procNode)) {
return false;
}
// make sure that either we're not clustered or this processor runs on all nodes or that this is the primary node
- shouldRun = !procNode.isIsolated() || !flowController.isClustered() || flowController.isPrimary();
- if (!shouldRun) {
+ if (!isRunOnCluster(procNode, flowController.isClustered(), flowController.isPrimary())) {
return false;
}
- // make sure that either proc has incoming FlowFiles or has no incoming connections or is annotated with @TriggerWhenEmpty
- shouldRun = procNode.isTriggerWhenEmpty() || !procNode.hasIncomingConnection() || Connectables.flowFilesQueued(procNode);
- if (!shouldRun) {
+ // Make sure processor has work to do. This means that it meets one of these criteria:
+ // * It is annotated with @TriggerWhenEmpty
+ // * It has data in an incoming Connection
+ // * It has no incoming connections
+ // * All incoming connections are self-loops
+ if (!isWorkToDo(procNode)) {
return true;
}
if (numRelationships > 0) {
final int requiredNumberOfAvailableRelationships = procNode.isTriggerWhenAnyDestinationAvailable() ? 1 : numRelationships;
- shouldRun = context.isRelationshipAvailabilitySatisfied(requiredNumberOfAvailableRelationships);
+ if (!context.isRelationshipAvailabilitySatisfied(requiredNumberOfAvailableRelationships)) {
+ return false;
+ }
}
final long batchNanos = procNode.getRunDuration(TimeUnit.NANOSECONDS);
@@ -112,10 +127,6 @@ public class ContinuallyRunProcessorTask implements Callable<Boolean> {
batch = false;
}
- if (!shouldRun) {
- return false;
- }
-
scheduleState.incrementActiveThreadCount();
final long startNanos = System.nanoTime();
@@ -123,6 +134,7 @@ public class ContinuallyRunProcessorTask implements Callable<Boolean> {
int invocationCount = 0;
try {
try (final AutoCloseable ncl = NarCloseable.withNarLoader()) {
+ boolean shouldRun = true;
while (shouldRun) {
procNode.onTrigger(processContext, sessionFactory);
invocationCount++;
@@ -135,10 +147,14 @@ public class ContinuallyRunProcessorTask implements Callable<Boolean> {
return false;
}
- shouldRun = procNode.isTriggerWhenEmpty() || !procNode.hasIncomingConnection() || Connectables.flowFilesQueued(procNode);
- shouldRun = shouldRun && (procNode.getYieldExpiration() < System.currentTimeMillis());
+ if (!isWorkToDo(procNode)) {
+ break;
+ }
+ if (isYielded(procNode)) {
+ break;
+ }
- if (shouldRun && numRelationships > 0) {
+ if (numRelationships > 0) {
final int requiredNumberOfAvailableRelationships = procNode.isTriggerWhenAnyDestinationAvailable() ? 1 : numRelationships;
shouldRun = context.isRelationshipAvailabilitySatisfied(requiredNumberOfAvailableRelationships);
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/69bce2c2/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/processor/StandardProcessContext.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/processor/StandardProcessContext.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/processor/StandardProcessContext.java
index 76849bd..a6302ca 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/processor/StandardProcessContext.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/processor/StandardProcessContext.java
@@ -32,6 +32,7 @@ import org.apache.nifi.controller.ControllerServiceLookup;
import org.apache.nifi.controller.ProcessorNode;
import org.apache.nifi.controller.service.ControllerServiceProvider;
import org.apache.nifi.encrypt.StringEncryptor;
+import org.apache.nifi.util.Connectables;
public class StandardProcessContext implements ProcessContext, ControllerServiceLookup {
@@ -185,6 +186,11 @@ public class StandardProcessContext implements ProcessContext, ControllerService
}
@Override
+ public boolean hasNonLoopConnection() {
+ return Connectables.hasNonLoopConnection(procNode);
+ }
+
+ @Override
public boolean hasConnection(Relationship relationship) {
Set<Connection> connections = procNode.getConnections(relationship);
return connections != null && !connections.isEmpty();
http://git-wip-us.apache.org/repos/asf/nifi/blob/69bce2c2/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/processor/StandardSchedulingContext.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/processor/StandardSchedulingContext.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/processor/StandardSchedulingContext.java
index 27d1264..a3c2a5d 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/processor/StandardSchedulingContext.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/processor/StandardSchedulingContext.java
@@ -118,6 +118,11 @@ public class StandardSchedulingContext implements SchedulingContext {
}
@Override
+ public boolean hasNonLoopConnection() {
+ return processContext.hasNonLoopConnection();
+ }
+
+ @Override
public boolean hasConnection(Relationship relationship) {
return processContext.hasConnection(relationship);
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/69bce2c2/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/util/Connectables.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/util/Connectables.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/util/Connectables.java
index c4d040b..5d74ebb 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/util/Connectables.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/util/Connectables.java
@@ -17,6 +17,7 @@
package org.apache.nifi.util;
import java.util.Collection;
+import java.util.List;
import org.apache.nifi.connectable.Connectable;
import org.apache.nifi.connectable.Connection;
@@ -53,4 +54,15 @@ public class Connectables {
return false;
}
+
+ public static boolean hasNonLoopConnection(final Connectable connectable) {
+ final List<Connection> connections = connectable.getIncomingConnections();
+ for (final Connection connection : connections) {
+ if (!connection.getSource().equals(connectable)) {
+ return true;
+ }
+ }
+
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/69bce2c2/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/tasks/TestContinuallyRunProcessorTask.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/tasks/TestContinuallyRunProcessorTask.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/tasks/TestContinuallyRunProcessorTask.java
new file mode 100644
index 0000000..174e5fb
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/tasks/TestContinuallyRunProcessorTask.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.controller.tasks;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.when;
+
+import java.util.Collections;
+
+import org.apache.nifi.connectable.Connection;
+import org.apache.nifi.controller.ProcessorNode;
+import org.apache.nifi.controller.queue.FlowFileQueue;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+public class TestContinuallyRunProcessorTask {
+
+ @Test
+ public void testIsWorkToDo() {
+ System.setProperty("nifi.properties.file.path", "src/test/resources/nifi.properties");
+
+ final ProcessorNode procNode = Mockito.mock(ProcessorNode.class);
+ Mockito.when(procNode.hasIncomingConnection()).thenReturn(false);
+
+ // There is work to do because there are no incoming connections.
+ assertTrue(ContinuallyRunProcessorTask.isWorkToDo(procNode));
+
+ // Test with only a single connection that is self-looping and empty
+ final Connection selfLoopingConnection = Mockito.mock(Connection.class);
+ when(selfLoopingConnection.getSource()).thenReturn(procNode);
+ when(selfLoopingConnection.getDestination()).thenReturn(procNode);
+
+ when(procNode.hasIncomingConnection()).thenReturn(true);
+ when(procNode.getIncomingConnections()).thenReturn(Collections.singletonList(selfLoopingConnection));
+ assertTrue(ContinuallyRunProcessorTask.isWorkToDo(procNode));
+
+ // Test with only a single connection that is self-looping and empty
+ final FlowFileQueue flowFileQueue = Mockito.mock(FlowFileQueue.class);
+ when(flowFileQueue.isActiveQueueEmpty()).thenReturn(true);
+
+ final FlowFileQueue nonEmptyQueue = Mockito.mock(FlowFileQueue.class);
+ when(nonEmptyQueue.isActiveQueueEmpty()).thenReturn(false);
+
+ when(selfLoopingConnection.getFlowFileQueue()).thenReturn(nonEmptyQueue);
+ assertTrue(ContinuallyRunProcessorTask.isWorkToDo(procNode));
+
+
+ // Test with only a non-looping Connection that has no FlowFiles
+ final Connection emptyConnection = Mockito.mock(Connection.class);
+ when(emptyConnection.getSource()).thenReturn(Mockito.mock(ProcessorNode.class));
+ when(emptyConnection.getDestination()).thenReturn(procNode);
+
+ when(emptyConnection.getFlowFileQueue()).thenReturn(flowFileQueue);
+ when(procNode.getIncomingConnections()).thenReturn(Collections.singletonList(emptyConnection));
+ assertFalse(ContinuallyRunProcessorTask.isWorkToDo(procNode));
+
+ // test when the queue has data
+ final Connection nonEmptyConnection = Mockito.mock(Connection.class);
+ when(nonEmptyConnection.getSource()).thenReturn(Mockito.mock(ProcessorNode.class));
+ when(nonEmptyConnection.getDestination()).thenReturn(procNode);
+ when(nonEmptyConnection.getFlowFileQueue()).thenReturn(nonEmptyQueue);
+ when(procNode.getIncomingConnections()).thenReturn(Collections.singletonList(nonEmptyConnection));
+ assertTrue(ContinuallyRunProcessorTask.isWorkToDo(procNode));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/69bce2c2/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ExecuteSQL.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ExecuteSQL.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ExecuteSQL.java
index 452df42..9aa9d59 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ExecuteSQL.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ExecuteSQL.java
@@ -129,7 +129,11 @@ public class ExecuteSQL extends AbstractProcessor {
FlowFile incoming = null;
if (context.hasIncomingConnection()) {
incoming = session.get();
- if (incoming == null) {
+
+ // If we have no FlowFile, and all incoming connections are self-loops then we can continue on.
+ // However, if we have no FlowFile and we have connections coming from other Processors, then
+ // we know that we should run only if we have a FlowFile.
+ if (incoming == null && context.hasNonLoopConnection()) {
return;
}
}
[24/50] [abbrv] nifi git commit: NIFI-1123: Fixing a botched commit.
Posted by mc...@apache.org.
NIFI-1123: Fixing a botched commit.
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/9e2f6df2
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/9e2f6df2
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/9e2f6df2
Branch: refs/heads/NIFI-655
Commit: 9e2f6df20511b814b761726e40f2d3b1f498cc9f
Parents: 52b24b9
Author: Tony Kurc <tr...@gmail.com>
Authored: Wed Nov 18 21:59:42 2015 -0500
Committer: Tony Kurc <tr...@gmail.com>
Committed: Wed Nov 18 21:59:42 2015 -0500
----------------------------------------------------------------------
.../org/apache/nifi/processors/attributes/UpdateAttribute.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/9e2f6df2/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-processor/src/main/java/org/apache/nifi/processors/attributes/UpdateAttribute.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-processor/src/main/java/org/apache/nifi/processors/attributes/UpdateAttribute.java b/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-processor/src/main/java/org/apache/nifi/processors/attributes/UpdateAttribute.java
index 9209f6d..cff2b55 100644
--- a/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-processor/src/main/java/org/apache/nifi/processors/attributes/UpdateAttribute.java
+++ b/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-processor/src/main/java/org/apache/nifi/processors/attributes/UpdateAttribute.java
@@ -133,7 +133,7 @@ public class UpdateAttribute extends AbstractProcessor implements Searchable {
private final Set<Relationship> relationships;
private static final Validator DELETE_PROPERTY_VALIDATOR = new Validator() {
- private static final Validator DPV_RE_VALIDATOR = StandardValidators.createRegexValidator(0, Integer.MAX_VALUE, true);
+ private final Validator DPV_RE_VALIDATOR = StandardValidators.createRegexValidator(0, Integer.MAX_VALUE, true);
@Override
public ValidationResult validate(String subject, String input, ValidationContext context) {
if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(input)) {
[23/50] [abbrv] nifi git commit: NIFI-1123 Adds expression language
support to DeleteAttributesExpression on UpdateAttributes Processor.
Posted by mc...@apache.org.
NIFI-1123 Adds expression language support to DeleteAttributesExpression on UpdateAttributes Processor.
Reviewed by Tony Kurc (trkurc@gmail.com) after Aldrin Piri <al...@apache.org> did the initial review and actionable comments
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/52b24b93
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/52b24b93
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/52b24b93
Branch: refs/heads/NIFI-655
Commit: 52b24b93d9f7763744c792c0cfed8974f8e6cb83
Parents: ab79403
Author: Joe Skora <js...@gmail.com>
Authored: Wed Nov 18 19:09:27 2015 -0500
Committer: Tony Kurc <tr...@gmail.com>
Committed: Wed Nov 18 19:10:21 2015 -0500
----------------------------------------------------------------------
.../processors/attributes/UpdateAttribute.java | 35 ++++-
.../update/attributes/TestUpdateAttribute.java | 130 +++++++++++++++++++
2 files changed, 163 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/52b24b93/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-processor/src/main/java/org/apache/nifi/processors/attributes/UpdateAttribute.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-processor/src/main/java/org/apache/nifi/processors/attributes/UpdateAttribute.java b/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-processor/src/main/java/org/apache/nifi/processors/attributes/UpdateAttribute.java
index 8cf5726..9209f6d 100644
--- a/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-processor/src/main/java/org/apache/nifi/processors/attributes/UpdateAttribute.java
+++ b/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-processor/src/main/java/org/apache/nifi/processors/attributes/UpdateAttribute.java
@@ -45,6 +45,7 @@ import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.PropertyValue;
import org.apache.nifi.components.ValidationContext;
import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.components.Validator;
import org.apache.nifi.expression.AttributeExpression;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.flowfile.attributes.CoreAttributes;
@@ -131,12 +132,40 @@ public class UpdateAttribute extends AbstractProcessor implements Searchable {
private final Set<Relationship> relationships;
+ private static final Validator DELETE_PROPERTY_VALIDATOR = new Validator() {
+ private static final Validator DPV_RE_VALIDATOR = StandardValidators.createRegexValidator(0, Integer.MAX_VALUE, true);
+ @Override
+ public ValidationResult validate(String subject, String input, ValidationContext context) {
+ if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(input)) {
+ final AttributeExpression.ResultType resultType = context.newExpressionLanguageCompiler().getResultType(input);
+ if (!resultType.equals(AttributeExpression.ResultType.STRING)) {
+ return new ValidationResult.Builder()
+ .subject(subject)
+ .input(input)
+ .valid(false)
+ .explanation("Expected property to to return type " + AttributeExpression.ResultType.STRING +
+ " but expression returns type " + resultType)
+ .build();
+ }
+ return new ValidationResult.Builder()
+ .subject(subject)
+ .input(input)
+ .valid(true)
+ .explanation("Property returns type " + AttributeExpression.ResultType.STRING)
+ .build();
+ }
+
+ return DPV_RE_VALIDATOR.validate(subject, input, context);
+ }
+ };
+
// static properties
public static final PropertyDescriptor DELETE_ATTRIBUTES = new PropertyDescriptor.Builder()
.name("Delete Attributes Expression")
.description("Regular expression for attributes to be deleted from flowfiles.")
.required(false)
- .addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR)
+ .addValidator(DELETE_PROPERTY_VALIDATOR)
+ .expressionLanguageSupported(true)
.build();
// relationships
@@ -477,7 +506,9 @@ public class UpdateAttribute extends AbstractProcessor implements Searchable {
}
} else {
try {
- final String regex = action.getValue();
+ final String actionValue = action.getValue();
+ final String regex = (actionValue == null) ? null :
+ getPropertyValue(actionValue, context).evaluateAttributeExpressions(flowfile).getValue();
if (regex != null) {
Pattern pattern = Pattern.compile(regex);
final Set<String> attributeKeys = flowfile.getAttributes().keySet();
http://git-wip-us.apache.org/repos/asf/nifi/blob/52b24b93/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-processor/src/test/java/org/apache/nifi/update/attributes/TestUpdateAttribute.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-processor/src/test/java/org/apache/nifi/update/attributes/TestUpdateAttribute.java b/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-processor/src/test/java/org/apache/nifi/update/attributes/TestUpdateAttribute.java
index f1b75ed..90b51bd 100644
--- a/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-processor/src/test/java/org/apache/nifi/update/attributes/TestUpdateAttribute.java
+++ b/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-processor/src/test/java/org/apache/nifi/update/attributes/TestUpdateAttribute.java
@@ -24,6 +24,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.UUID;
+import java.util.regex.PatternSyntaxException;
import org.apache.nifi.processors.attributes.UpdateAttribute;
import org.apache.nifi.update.attributes.serde.CriteriaSerDe;
@@ -33,6 +34,8 @@ import org.apache.nifi.util.TestRunners;
import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+
/**
*
*/
@@ -500,4 +503,131 @@ public class TestUpdateAttribute {
result.get(0).assertAttributeNotExists("sample.1");
result.get(0).assertAttributeExists("simple.1");
}
+
+ @Test
+ public void testAttributeKey() {
+ final TestRunner runner = TestRunners.newTestRunner(new UpdateAttribute());
+ runner.setProperty("Delete Attributes Expression", "(attribute\\.[2-5]|sample.*)");
+
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("attribute.1", "value.1");
+ attributes.put("attribute.2", "value.2");
+ attributes.put("attribute.6", "value.6");
+ attributes.put("sampleSize", "value.size");
+ attributes.put("sample.1", "value.sample.1");
+ attributes.put("simple.1", "value.simple.1");
+ runner.enqueue(new byte[0], attributes);
+
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred(UpdateAttribute.REL_SUCCESS, 1);
+ final List<MockFlowFile> result = runner.getFlowFilesForRelationship(UpdateAttribute.REL_SUCCESS);
+ result.get(0).assertAttributeEquals("attribute.1", "value.1");
+ result.get(0).assertAttributeNotExists("attribute.2");
+ result.get(0).assertAttributeExists("attribute.6");
+ result.get(0).assertAttributeNotExists("sampleSize");
+ result.get(0).assertAttributeNotExists("sample.1");
+ result.get(0).assertAttributeExists("simple.1");
+ }
+
+ @Test
+ public void testExpressionLiteralDelete() {
+ final TestRunner runner = TestRunners.newTestRunner(new UpdateAttribute());
+ runner.setProperty("Delete Attributes Expression", "${literal('attribute\\.'):append(${literal(6)})}");
+
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("attribute.1", "value.1");
+ attributes.put("attribute.2", "value.2");
+ attributes.put("attribute.6", "value.6");
+ attributes.put("sampleSize", "value.size");
+ attributes.put("sample.1", "value.sample.1");
+ attributes.put("simple.1", "value.simple.1");
+ runner.enqueue(new byte[0], attributes);
+
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred(UpdateAttribute.REL_SUCCESS, 1);
+ final List<MockFlowFile> result = runner.getFlowFilesForRelationship(UpdateAttribute.REL_SUCCESS);
+ result.get(0).assertAttributeEquals("attribute.1", "value.1");
+ result.get(0).assertAttributeExists("attribute.2");
+ result.get(0).assertAttributeNotExists("attribute.6");
+ result.get(0).assertAttributeExists("sampleSize");
+ result.get(0).assertAttributeExists("sample.1");
+ result.get(0).assertAttributeExists("simple.1");
+ }
+
+ @Test
+ public void testExpressionRegexDelete() {
+ final TestRunner runner = TestRunners.newTestRunner(new UpdateAttribute());
+ runner.setProperty("Delete Attributes Expression", "${literal('(attribute\\.'):append(${literal('[2-5]')}):append(${literal('|sample.*)')})}");
+
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("attribute.1", "value.1");
+ attributes.put("attribute.2", "value.2");
+ attributes.put("attribute.6", "value.6");
+ attributes.put("sampleSize", "value.size");
+ attributes.put("sample.1", "value.sample.1");
+ attributes.put("simple.1", "value.simple.1");
+ runner.enqueue(new byte[0], attributes);
+
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred(UpdateAttribute.REL_SUCCESS, 1);
+ final List<MockFlowFile> result = runner.getFlowFilesForRelationship(UpdateAttribute.REL_SUCCESS);
+ result.get(0).assertAttributeEquals("attribute.1", "value.1");
+ result.get(0).assertAttributeNotExists("attribute.2");
+ result.get(0).assertAttributeExists("attribute.6");
+ result.get(0).assertAttributeNotExists("sampleSize");
+ result.get(0).assertAttributeNotExists("sample.1");
+ result.get(0).assertAttributeExists("simple.1");
+ }
+
+ @Test
+ public void testAttributeListDelete() {
+ final TestRunner runner = TestRunners.newTestRunner(new UpdateAttribute());
+ runner.setProperty("Delete Attributes Expression", "attribute.1|attribute.2|sample.1|simple.1");
+
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("attribute.1", "value.1");
+ attributes.put("attribute.2", "value.2");
+ attributes.put("attribute.6", "value.6");
+ attributes.put("sampleSize", "value.size");
+ attributes.put("sample.1", "value.sample.1");
+ attributes.put("simple.1", "value.simple.1");
+ runner.enqueue(new byte[0], attributes);
+
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred(UpdateAttribute.REL_SUCCESS, 1);
+ final List<MockFlowFile> result = runner.getFlowFilesForRelationship(UpdateAttribute.REL_SUCCESS);
+ result.get(0).assertAttributeNotExists("attribute.1");
+ result.get(0).assertAttributeNotExists("attribute.2");
+ result.get(0).assertAttributeExists("attribute.6");
+ result.get(0).assertAttributeExists("sampleSize");
+ result.get(0).assertAttributeNotExists("sample.1");
+ result.get(0).assertAttributeNotExists("simple.1");
+ }
+
+ @Test
+ public void testInvalidRegex() {
+ final TestRunner runner = TestRunners.newTestRunner(new UpdateAttribute());
+ runner.setProperty("Delete Attributes Expression", "(");
+ runner.assertNotValid();
+ }
+
+ @Test
+ public void testInvalidRegexInAttribute() {
+ final TestRunner runner = TestRunners.newTestRunner(new UpdateAttribute());
+ runner.setProperty("Delete Attributes Expression", "${butter}");
+ runner.assertValid();
+
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("butter", "(");
+ runner.enqueue(new byte[0], attributes);
+ try {
+ runner.run();
+ } catch (Throwable t) {
+ assertEquals(t.getCause().getClass(), PatternSyntaxException.class);
+ }
+ }
}
[11/50] [abbrv] nifi git commit: NIFI-1178: Updates to Administration
Guide - System Properties section.
Posted by mc...@apache.org.
NIFI-1178: Updates to Administration Guide - System Properties section.
Signed-off-by: Mark Payne <ma...@hotmail.com>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/f98e335c
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/f98e335c
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/f98e335c
Branch: refs/heads/NIFI-655
Commit: f98e335cdec4e32fe41b8abaef8118cba7f132d1
Parents: 03a54bf
Author: Jenn Barnabee <je...@gmail.com>
Authored: Tue Nov 17 04:11:57 2015 -0500
Committer: Mark Payne <ma...@hotmail.com>
Committed: Tue Nov 17 08:16:45 2015 -0500
----------------------------------------------------------------------
.../src/main/asciidoc/administration-guide.adoc | 16 +++++++++++-----
1 file changed, 11 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/f98e335c/nifi-docs/src/main/asciidoc/administration-guide.adoc
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/administration-guide.adoc b/nifi-docs/src/main/asciidoc/administration-guide.adoc
index be7b21e..57cee4e 100644
--- a/nifi-docs/src/main/asciidoc/administration-guide.adoc
+++ b/nifi-docs/src/main/asciidoc/administration-guide.adoc
@@ -381,8 +381,8 @@ Bootstrap Properties
The _bootstrap.conf_ file in the _conf_ directory allows users to configure settings for how NiFi should be started.
This includes parameters, such as the size of the Java Heap, what Java command to run, and Java System Properties.
-Here, we will address the different properties that are made available in the file. Any chances to this file will
-take affect only after NiFi has been stopped and restarted.
+Here, we will address the different properties that are made available in the file. Any changes to this file will
+take effect only after NiFi has been stopped and restarted.
|====
|*Property*|*Description*
@@ -509,16 +509,14 @@ The first section of the _nifi.properties_ file is for the Core Properties. Thes
|nifi.flowcontroller.graceful.shutdown.period|Indicates the shutdown period. The default value is 10 sec.
|nifi.flowservice.writedelay.interval|When many changes are made to the flow.xml, this property specifies how long to wait before writing out the changes, so as to batch the changes into a single write. The default value is 500 ms.
|nifi.administrative.yield.duration|If a component allows an unexpected exception to escape, it is considered a bug. As a result, the framework will pause (or administratively yield) the component for this amount of time. This is done so that the component does not use up massive amounts of system resources, since it is known to have problems in the existing state. The default value is 30 sec.
+|nifi.bored.yield.duration|When a component has no work to do (i.e., is "bored"), this is the amount of time it will wait before checking to see if it has new data to work on. This way, it does not use up CPU resources by checking for new work too often. When setting this property, be aware that it could add extra latency for components that do not constantly have work to do, as once they go into this "bored" state, they will wait this amount of time before checking for more work. The default value is 10 millis.
|nifi.authority.provider.configuration.file*|This is the location of the file that specifies how user access is authenticated. The default value is ./conf/authority-providers.xml.
-|nifi.reporting.task.configuration.file*|This is the location of the Reporting Tasks file. The default value is ./conf/reporting-tasks.xml.
-|nifi.controller.service.configuration.file*|This is the location of the Controller Services file. The default value is ./conf/controller-services.xml.
|nifi.templates.directory*|This is the location of the directory where flow templates are saved. The default value is ./conf/templates.l
|nifi.ui.banner.text|This is banner text that may be configured to display at the top of the User Interface. It is blank by default.
|nifi.ui.autorefresh.interval|The interval at which the User Interface auto-refreshes. The default value is 30 sec.
|nifi.nar.library.directory|The location of the nar library. The default value is ./lib and probably should be left as is.
|nifi.nar.working.directory|The location of the nar working directory. The default value is ./work/nar and probably should be left as is.
|nifi.documentation.working.directory|The documentation working directory. The default value is ./work/docs/components and probably should be left as is.
-|nifi.bored.yield.duration|When a component has no work to do (i.e., is "bored"), this is the amount of time it will wait before checking to see if it has new data to work on. This way, it does not use up CPU resources by checking for new work too often. When setting this property, be aware that it could add extra latency for components that do not constantly have work to do, as once they go into this "bored" state, they will wait this amount of time before checking for more work. The default value is 10 millis.
|====
*H2 Settings* +
@@ -633,6 +631,7 @@ Providing three total locations, including _nifi.provenance.repository.director
|nifi.provenance.repository.indexed.fields|This is a comma-separated list of the fields that should be indexed and made searchable. Fields that are not indexed will not be searchable. Valid fields are: EventType, FlowFileUUID, Filename, TransitURI, ProcessorID, AlternateIdentifierURI, ContentType, Relationship, Details. The default value is: EventType, FlowFileUUID, Filename, ProcessorID.
|nifi.provenance.repository.indexed.attributes|This is a comma-separated list of FlowFile Attributes that should be indexed and made searchable. It is blank by default.
|nifi.provenance.repository.index.shard.size|Large values for the shard size will result in more Java heap usage when searching the Provenance Repository but should provide better performance. The default value is 500 MB.
+|nifi.provenance.repository.max.attribute.length|Indicates the maximum length that a FlowFile attribute can be when retrieving a Provenance Event from the repository. If the length of any attribute exceeds this value, it will be truncated when the event is retrieved. The default is 65536.
|====
*Volatile Provenance Repository Properties* +
@@ -766,6 +765,13 @@ Only configure these properties for the cluster manager.
|nifi.cluster.manager.safemode.duration|Upon restart of an already existing cluster, this is the amount of time that the cluster manager waits for the primary node to connect before giving up and selecting another node to be the primary node. The default value is 0 sec, which means to wait forever. If the administrator does not care which node is the primary node, this value can be changed to some amount of time other than 0 sec.
|====
+*Kerberos* +
+
+|====
+|*Property*|*Description*
+|nifi.kerberos.krb5.file*|The location of the krb5 file, if used. It is blank by default.
+|====
+
NOTE: *For Upgrading* - Take care when configuring the properties above that are marked with an asterisk (*). To make the upgrade process easier, it is advisable to change the default configurations to locations outside the main root installation directory. In this way, these items can remain in their configured location through an upgrade, and NiFi can find all the repositories and configuration files and pick up where it left off as soon as the old version is stopped and the new version is started. Furthermore, the administrator may reuse this _nifi.properties_ file and any other configuration files without having to re-configure them each time an upgrade takes place. As previously noted, it is important to check for any changes in the _nifi.properties_ file of the new version when upgrading and make sure they are reflected in the _nifi.properties_ file you use.
[09/50] [abbrv] nifi git commit: NIFI-1146 Allow GetKafka to be
configured with auto.offset.reset to largest or smallest
Posted by mc...@apache.org.
NIFI-1146 Allow GetKafka to be configured with auto.offset.reset to largest or smallest
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/b954ca62
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/b954ca62
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/b954ca62
Branch: refs/heads/NIFI-655
Commit: b954ca620e619e7961e6a7a58122b844f30862da
Parents: 64369f6
Author: Naveen Madhire <na...@capitalone.com>
Authored: Mon Nov 16 11:59:52 2015 -0600
Committer: Naveen Madhire <na...@capitalone.com>
Committed: Mon Nov 16 11:59:52 2015 -0600
----------------------------------------------------------------------
.../apache/nifi/processors/kafka/GetKafka.java | 52 +++++++++++++-------
1 file changed, 33 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/b954ca62/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/processors/kafka/GetKafka.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/processors/kafka/GetKafka.java b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/processors/kafka/GetKafka.java
index e10977b..e644064 100644
--- a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/processors/kafka/GetKafka.java
+++ b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/processors/kafka/GetKafka.java
@@ -65,13 +65,18 @@ import kafka.message.MessageAndMetadata;
@CapabilityDescription("Fetches messages from Apache Kafka")
@Tags({"Kafka", "Apache", "Get", "Ingest", "Ingress", "Topic", "PubSub"})
@WritesAttributes({
- @WritesAttribute(attribute = "kafka.topic", description = "The name of the Kafka Topic from which the message was received"),
- @WritesAttribute(attribute = "kafka.key", description = "The key of the Kafka message, if it exists and batch size is 1. If"
- + " the message does not have a key, or if the batch size is greater than 1, this attribute will not be added"),
- @WritesAttribute(attribute = "kafka.partition", description = "The partition of the Kafka Topic from which the message was received. This attribute is added only if the batch size is 1"),
- @WritesAttribute(attribute = "kafka.offset", description = "The offset of the message within the Kafka partition. This attribute is added only if the batch size is 1")})
+ @WritesAttribute(attribute = "kafka.topic", description = "The name of the Kafka Topic from which the message was received"),
+ @WritesAttribute(attribute = "kafka.key", description = "The key of the Kafka message, if it exists and batch size is 1. If"
+ + " the message does not have a key, or if the batch size is greater than 1, this attribute will not be added"),
+ @WritesAttribute(attribute = "kafka.partition", description = "The partition of the Kafka Topic from which the message was received. This attribute is added only if the batch size is 1"),
+ @WritesAttribute(attribute = "kafka.offset", description = "The offset of the message within the Kafka partition. This attribute is added only if the batch size is 1"),
+ @WritesAttribute(attribute = "auto.offset.reset", description = "If this is set to largest, the consumer may lose some messages when the number of partitions, " +
+ "for the topics it subscribes to, changes on the broker. To prevent data loss during partition addition, set auto.offset.reset to smallest")})
public class GetKafka extends AbstractProcessor {
+ public static final String SMALLEST = "smallest";
+ public static final String LARGEST = "largest";
+
public static final PropertyDescriptor ZOOKEEPER_CONNECTION_STRING = new PropertyDescriptor.Builder()
.name("ZooKeeper Connection String")
.description("The Connection String to use in order to connect to ZooKeeper. This is often a comma-separated list of <host>:<port>"
@@ -141,12 +146,20 @@ public class GetKafka extends AbstractProcessor {
.expressionLanguageSupported(false)
.build();
public static final PropertyDescriptor GROUP_ID = new PropertyDescriptor.Builder()
- .name("Group ID")
- .description("A Group ID is used to identify consumers that are within the same consumer group")
- .required(true)
- .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
- .expressionLanguageSupported(false)
- .build();
+ .name("Group ID")
+ .description("A Group ID is used to identify consumers that are within the same consumer group")
+ .required(true)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .expressionLanguageSupported(false)
+ .build();
+
+ public static final PropertyDescriptor AUTO_OFFSET_RESET = new PropertyDescriptor.Builder()
+ .name("Auto Offset Reset")
+ .description("Auto Offset Reset indicator")
+ .required(true)
+ .allowableValues(SMALLEST, LARGEST)
+ .defaultValue(LARGEST)
+ .build();
public static final Relationship REL_SUCCESS = new Relationship.Builder()
.name("success")
@@ -163,13 +176,13 @@ public class GetKafka extends AbstractProcessor {
@Override
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
final PropertyDescriptor clientNameWithDefault = new PropertyDescriptor.Builder()
- .fromPropertyDescriptor(CLIENT_NAME)
- .defaultValue("NiFi-" + getIdentifier())
- .build();
+ .fromPropertyDescriptor(CLIENT_NAME)
+ .defaultValue("NiFi-" + getIdentifier())
+ .build();
final PropertyDescriptor groupIdWithDefault = new PropertyDescriptor.Builder()
- .fromPropertyDescriptor(GROUP_ID)
- .defaultValue(getIdentifier())
- .build();
+ .fromPropertyDescriptor(GROUP_ID)
+ .defaultValue(getIdentifier())
+ .build();
final List<PropertyDescriptor> props = new ArrayList<>();
props.add(ZOOKEEPER_CONNECTION_STRING);
@@ -181,6 +194,7 @@ public class GetKafka extends AbstractProcessor {
props.add(groupIdWithDefault);
props.add(KAFKA_TIMEOUT);
props.add(ZOOKEEPER_TIMEOUT);
+ props.add(AUTO_OFFSET_RESET);
return props;
}
@@ -204,7 +218,7 @@ public class GetKafka extends AbstractProcessor {
props.setProperty("client.id", context.getProperty(CLIENT_NAME).getValue());
props.setProperty("auto.commit.interval.ms", String.valueOf(context.getProperty(ZOOKEEPER_COMMIT_DELAY).asTimePeriod(TimeUnit.MILLISECONDS)));
props.setProperty("auto.commit.enable", "true"); // just be explicit
- props.setProperty("auto.offset.reset", "smallest");
+ props.setProperty("auto.offset.reset", context.getProperty(AUTO_OFFSET_RESET).getValue());
props.setProperty("zk.connectiontimeout.ms", context.getProperty(ZOOKEEPER_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).toString());
props.setProperty("socket.timeout.ms", context.getProperty(KAFKA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).toString());
@@ -356,4 +370,4 @@ public class GetKafka extends AbstractProcessor {
}
}
-}
+}
\ No newline at end of file
[38/50] [abbrv] nifi git commit: NIFI-1171: Ensure that we pick up
changes when files roll over and ensure that we don't pick up the rolled over
file multiple times
Posted by mc...@apache.org.
NIFI-1171: Ensure that we pick up changes when files roll over and ensure that we don't pick up the rolled over file multiple times
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/2516b1da
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/2516b1da
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/2516b1da
Branch: refs/heads/NIFI-655
Commit: 2516b1dad24e39efaeac49a244eea42847602e06
Parents: 8c2323d
Author: Mark Payne <ma...@hotmail.com>
Authored: Mon Nov 23 11:59:23 2015 -0500
Committer: Mark Payne <ma...@hotmail.com>
Committed: Mon Nov 23 14:11:14 2015 -0500
----------------------------------------------------------------------
.../nifi/processors/standard/TailFile.java | 682 ++++++++++---------
.../nifi/processors/standard/TestTailFile.java | 151 +++-
2 files changed, 487 insertions(+), 346 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/2516b1da/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TailFile.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TailFile.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TailFile.java
index 1c32c00..3d6d3a0 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TailFile.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TailFile.java
@@ -47,8 +47,8 @@ import java.util.zip.Checksum;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.annotation.behavior.InputRequirement;
-import org.apache.nifi.annotation.behavior.TriggerSerially;
import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.TriggerSerially;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.annotation.lifecycle.OnScheduled;
@@ -71,12 +71,14 @@ import org.apache.nifi.util.LongHolder;
// note: it is important that this Processor is not marked as @SupportsBatching because the session commits must complete before persisting state locally; otherwise, data loss may occur
@TriggerSerially
+@InputRequirement(Requirement.INPUT_FORBIDDEN)
@Tags({"tail", "file", "log", "text", "source"})
@CapabilityDescription("\"Tails\" a file, ingesting data from the file as it is written to the file. The file is expected to be textual. Data is ingested only when a "
+ "new line is encountered (carriage return or new-line character or combination). If the file to tail is periodically \"rolled over\", as is generally the case "
+ "with log files, an optional Rolling Filename Pattern can be used to retrieve data from files that have rolled over, even if the rollover occurred while NiFi "
- + "was not running (provided that the data still exists upon restart of NiFi).")
-@InputRequirement(Requirement.INPUT_FORBIDDEN)
+ + "was not running (provided that the data still exists upon restart of NiFi). It is generally advisable to set the Run Schedule to a few seconds, rather than running "
+ + "with the default value of 0 secs, as this Processor will consume a lot of resources if scheduled very aggressively. At this time, this Processor does not support "
+ + "ingesting files that have been compressed when 'rolled over'.")
public class TailFile extends AbstractProcessor {
static final AllowableValue START_BEGINNING_OF_TIME = new AllowableValue("Beginning of Time", "Beginning of Time",
@@ -124,7 +126,6 @@ public class TailFile extends AbstractProcessor {
.build();
private volatile TailFileState state = new TailFileState(null, null, null, 0L, 0L, null, ByteBuffer.allocate(65536));
- private volatile boolean recoveredRolledFiles = false;
private volatile Long expectedRecoveryChecksum;
private volatile boolean tailFileChanged = false;
@@ -147,17 +148,12 @@ public class TailFile extends AbstractProcessor {
public void onPropertyModified(final PropertyDescriptor descriptor, final String oldValue, final String newValue) {
if (FILENAME.equals(descriptor)) {
state = new TailFileState(newValue, null, null, 0L, 0L, null, ByteBuffer.allocate(65536));
- recoveredRolledFiles = false;
tailFileChanged = true;
- } else if (ROLLING_FILENAME_PATTERN.equals(descriptor)) {
- recoveredRolledFiles = false;
}
}
@OnScheduled
public void recoverState(final ProcessContext context) throws IOException {
- recoveredRolledFiles = false;
-
final String tailFilename = context.getProperty(FILENAME).getValue();
final String stateFilename = context.getProperty(STATE_FILE).getValue();
@@ -172,7 +168,7 @@ public class TailFile extends AbstractProcessor {
if (encodingVersion == 0) {
final String filename = dis.readUTF();
- final long position = dis.readLong();
+ long position = dis.readLong();
final long timestamp = dis.readLong();
final boolean checksumPresent = dis.readBoolean();
@@ -202,18 +198,25 @@ public class TailFile extends AbstractProcessor {
getLogger().debug("When recovering state, checksum of tailed file matches the stored checksum. Will resume where left off.");
tailFile = existingTailFile;
reader = FileChannel.open(tailFile.toPath(), StandardOpenOption.READ);
- getLogger().debug("Created RandomAccessFile {} for {} in recoverState", new Object[] {reader, tailFile});
+ getLogger().debug("Created FileChannel {} for {} in recoverState", new Object[] {reader, tailFile});
reader.position(position);
} else {
+ // we don't seek the reader to the position, so our reader will start at beginning of file.
getLogger().debug("When recovering state, checksum of tailed file does not match the stored checksum. Will begin tailing current file from beginning.");
}
}
+ } else {
+ // fewer bytes than our position, so we know we weren't already reading from this file. Keep reader at a position of 0.
+ getLogger().debug("When recovering state, existing file to tail is only {} bytes but position flag is {}; "
+ + "this indicates that the file has rotated. Will begin tailing current file from beginning.", new Object[] {existingTailFile.length(), position});
}
state = new TailFileState(tailFilename, tailFile, reader, position, timestamp, checksum, ByteBuffer.allocate(65536));
} else {
+ // If filename changed or there is no checksum present, then we have no expected checksum to use for recovery.
expectedRecoveryChecksum = null;
+
// tailing a new file since the state file was written out. We will reset state.
state = new TailFileState(tailFilename, null, null, 0L, 0L, null, ByteBuffer.allocate(65536));
}
@@ -234,224 +237,79 @@ public class TailFile extends AbstractProcessor {
return;
}
- final FileChannel channel = state.getReader();
- if (channel == null) {
+ final FileChannel reader = state.getReader();
+ if (reader == null) {
return;
}
try {
- channel.close();
+ reader.close();
} catch (final IOException ioe) {
getLogger().warn("Failed to close file handle during cleanup");
}
- getLogger().debug("Closed FileChannel {}", new Object[] {channel});
+ getLogger().debug("Closed FileChannel {}", new Object[] {reader});
this.state = new TailFileState(state.getFilename(), state.getFile(), null, state.getPosition(), state.getTimestamp(), state.getChecksum(), state.getBuffer());
}
- public void persistState(final TailFileState state, final String stateFilename) throws IOException {
- getLogger().debug("Persisting state {} to {}", new Object[] {state, stateFilename});
- final File stateFile = new File(stateFilename);
- File directory = stateFile.getParentFile();
- if (directory != null && !directory.exists() && !directory.mkdirs()) {
- getLogger().warn("Failed to persist state to {} because the parent directory does not exist and could not be created. This may result in data being duplicated upon restart of NiFi");
- return;
- }
- try (final FileOutputStream fos = new FileOutputStream(stateFile);
- final DataOutputStream dos = new DataOutputStream(fos)) {
-
- dos.writeInt(0); // version
- dos.writeUTF(state.getFilename());
- dos.writeLong(state.getPosition());
- dos.writeLong(state.getTimestamp());
- if (state.getChecksum() == null) {
- dos.writeBoolean(false);
+ @Override
+ public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
+ // If user changes the file that is being tailed, we need to consume the already-rolled-over data according
+ // to the Initial Start Position property
+ boolean rolloverOccurred;
+ if (tailFileChanged) {
+ rolloverOccurred = false;
+ final String recoverPosition = context.getProperty(START_POSITION).getValue();
+
+ if (START_BEGINNING_OF_TIME.getValue().equals(recoverPosition)) {
+ recoverRolledFiles(context, session, this.expectedRecoveryChecksum, state.getTimestamp(), state.getPosition());
+ } else if (START_CURRENT_FILE.getValue().equals(recoverPosition)) {
+ cleanup();
+ state = new TailFileState(context.getProperty(FILENAME).getValue(), null, null, 0L, 0L, null, state.getBuffer());
} else {
- dos.writeBoolean(true);
- dos.writeLong(state.getChecksum().getValue());
- }
- }
- }
-
- private FileChannel createReader(final File file, final long position) {
- final FileChannel reader;
-
- try {
- reader = FileChannel.open(file.toPath(), StandardOpenOption.READ);
- } catch (final IOException ioe) {
- getLogger().warn("Could not open {} due to {}; will attempt to access file again after the configured Yield Duration has elapsed", new Object[] {file, ioe});
- return null;
- }
+ final String filename = context.getProperty(FILENAME).getValue();
+ final File file = new File(filename);
- getLogger().debug("Created RandomAccessFile {} for {}", new Object[] {reader, file});
+ try {
+ final FileChannel fileChannel = FileChannel.open(file.toPath(), StandardOpenOption.READ);
+ getLogger().debug("Created FileChannel {} for {}", new Object[] {fileChannel, file});
- try {
- reader.position(position);
- } catch (final IOException ioe) {
- getLogger().error("Failed to read from {} due to {}", new Object[] {file, ioe});
-
- try {
- reader.close();
- getLogger().debug("Closed RandomAccessFile {}", new Object[] {reader});
- } catch (final IOException ioe2) {
- }
-
- return null;
- }
-
- return reader;
- }
-
- // for testing purposes
- TailFileState getState() {
- return state;
- }
-
-
-
- /**
- * Finds any files that have rolled over and have not yet been ingested by this Processor. Each of these files that is found will be
- * ingested as its own FlowFile. If a file is found that has been partially ingested, the rest of the file will be ingested as a
- * single FlowFile but the data that already has been ingested will not be ingested again.
- *
- * @param context the ProcessContext to use in order to obtain Processor configuration
- * @param session the ProcessSession to use in order to interact with FlowFile creation and content.
- */
- private void recoverRolledFiles(final ProcessContext context, final ProcessSession session) {
- try {
- // Find all files that match our rollover pattern, if any, and order them based on their timestamp and filename.
- // Ignore any file that has a timestamp earlier than the state that we have persisted. If we were reading from
- // a file when we stopped running, then that file that we were reading from should be the first file in this list,
- // assuming that the file still exists on the file system.
- final List<File> rolledOffFiles = getRolledOffFiles(context, state.getTimestamp());
- getLogger().debug("Recovering Rolled Off Files; total number of files rolled off = {}", new Object[] {rolledOffFiles.size()});
-
- // For first file that we find, it may or may not be the file that we were last reading from.
- // As a result, we have to read up to the position we stored, while calculating the checksum. If the checksums match,
- // then we know we've already processed this file. If the checksums do not match, then we have not
- // processed this file and we need to seek back to position 0 and ingest the entire file.
- // For all other files that have been rolled over, we need to just ingest the entire file.
- if (!rolledOffFiles.isEmpty() && expectedRecoveryChecksum != null && rolledOffFiles.get(0).length() >= state.getPosition()) {
- final File firstFile = rolledOffFiles.get(0);
-
- final long startNanos = System.nanoTime();
- try (final InputStream fis = new FileInputStream(firstFile);
- final CheckedInputStream in = new CheckedInputStream(fis, new CRC32())) {
- StreamUtils.copy(in, new NullOutputStream(), state.getPosition());
-
- final long checksumResult = in.getChecksum().getValue();
- if (checksumResult == expectedRecoveryChecksum) {
- getLogger().debug("Checksum for {} matched expected checksum. Will skip first {} bytes", new Object[] {firstFile, state.getPosition()});
-
- // This is the same file that we were reading when we shutdown. Start reading from this point on.
- rolledOffFiles.remove(0);
- FlowFile flowFile = session.create();
- flowFile = session.importFrom(in, flowFile);
- if (flowFile.getSize() == 0L) {
- session.remove(flowFile);
- // use a timestamp of lastModified() + 1 so that we do not ingest this file again.
- cleanup();
- state = new TailFileState(context.getProperty(FILENAME).getValue(), null, null, 0L, firstFile.lastModified() + 1L, null, state.getBuffer());
- } else {
- flowFile = session.putAttribute(flowFile, "filename", firstFile.getName());
-
- session.getProvenanceReporter().receive(flowFile, firstFile.toURI().toString(), "FlowFile contains bytes 0 through " + state.getPosition() + " of source file",
- TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos));
- session.transfer(flowFile, REL_SUCCESS);
-
- // use a timestamp of lastModified() + 1 so that we do not ingest this file again.
- cleanup();
- state = new TailFileState(context.getProperty(FILENAME).getValue(), null, null, 0L, firstFile.lastModified() + 1L, null, state.getBuffer());
+ final Checksum checksum = new CRC32();
+ final long position = file.length();
+ final long timestamp = file.lastModified();
- // must ensure that we do session.commit() before persisting state in order to avoid data loss.
- session.commit();
- persistState(state, context.getProperty(STATE_FILE).getValue());
- }
- } else {
- getLogger().debug("Checksum for {} did not match expected checksum. Checksum for file was {} but expected {}. Will consume entire file",
- new Object[] {firstFile, checksumResult, expectedRecoveryChecksum});
+ try (final InputStream fis = new FileInputStream(file);
+ final CheckedInputStream in = new CheckedInputStream(fis, checksum)) {
+ StreamUtils.copy(in, new NullOutputStream(), position);
}
- }
- }
-
- // For each file that we found that matches our Rollover Pattern, and has a last modified date later than the timestamp
- // that we recovered from the state file, we need to consume the entire file. The only exception to this is the file that
- // we were reading when we last stopped, as it may already have been partially consumed. That is taken care of in the
- // above block of code.
- for (final File file : rolledOffFiles) {
- FlowFile flowFile = session.create();
- flowFile = session.importFrom(file.toPath(), true, flowFile);
- if (flowFile.getSize() == 0L) {
- session.remove(flowFile);
- } else {
- flowFile = session.putAttribute(flowFile, "filename", file.getName());
- session.getProvenanceReporter().receive(flowFile, file.toURI().toString());
- session.transfer(flowFile, REL_SUCCESS);
- // use a timestamp of lastModified() + 1 so that we do not ingest this file again.
+ fileChannel.position(position);
cleanup();
- state = new TailFileState(context.getProperty(FILENAME).getValue(), null, null, 0L, file.lastModified() + 1L, null, state.getBuffer());
-
- // must ensure that we do session.commit() before persisting state in order to avoid data loss.
- session.commit();
- persistState(state, context.getProperty(STATE_FILE).getValue());
+ state = new TailFileState(filename, file, fileChannel, position, timestamp, checksum, state.getBuffer());
+ } catch (final IOException ioe) {
+ getLogger().error("Attempted to position Reader at current position in file {} but failed to do so due to {}", new Object[] {file, ioe.toString()}, ioe);
+ context.yield();
+ return;
}
}
- } catch (final IOException e) {
- getLogger().error("Failed to recover files that have rolled over due to {}", new Object[] {e});
- }
- }
-
-
- @Override
- public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
- // If this is the first time the processor has run since it was started, we need to check for any files that may have rolled over
- // while the processor was stopped. If we find any, we need to import them into the flow.
- if (!recoveredRolledFiles) {
- if (tailFileChanged) {
- final String recoverPosition = context.getProperty(START_POSITION).getValue();
-
- if (START_BEGINNING_OF_TIME.getValue().equals(recoverPosition)) {
- recoverRolledFiles(context, session);
- } else if (START_CURRENT_FILE.getValue().equals(recoverPosition)) {
- cleanup();
- state = new TailFileState(context.getProperty(FILENAME).getValue(), null, null, 0L, 0L, null, state.getBuffer());
- } else {
- final String filename = context.getProperty(FILENAME).getValue();
- final File file = new File(filename);
-
- try {
- final FileChannel channel = FileChannel.open(file.toPath(), StandardOpenOption.READ);
- getLogger().debug("Created FileChannel {} for {}", new Object[] {channel, file});
- final Checksum checksum = new CRC32();
- final long position = file.length();
- final long timestamp = file.lastModified();
-
- try (final InputStream fis = new FileInputStream(file);
- final CheckedInputStream in = new CheckedInputStream(fis, checksum)) {
- StreamUtils.copy(in, new NullOutputStream(), position);
- }
-
- channel.position(position);
- cleanup();
- state = new TailFileState(filename, file, channel, position, timestamp, checksum, state.getBuffer());
- } catch (final IOException ioe) {
- getLogger().error("Attempted to position Reader at current position in file {} but failed to do so due to {}", new Object[] {file, ioe.toString()}, ioe);
- context.yield();
- return;
- }
- }
-
- tailFileChanged = false;
- } else {
- recoverRolledFiles(context, session);
+ tailFileChanged = false;
+ } else {
+ // Recover any data that may have rolled over since the last time that this processor ran.
+ // If expectedRecoveryChecksum != null, that indicates that this is the first iteration since processor was started, so use whatever checksum value
+ // was present when the state was last persisted. In this case, we must then null out the value so that the next iteration won't keep using the "recovered"
+ // value. If the value is null, then we know that either the processor has already recovered that data, or there was no state persisted. In either case,
+ // use whatever checksum value is currently in the state.
+ Long expectedChecksumValue = expectedRecoveryChecksum;
+ if (expectedChecksumValue == null) {
+ expectedChecksumValue = state.getChecksum() == null ? null : state.getChecksum().getValue();
}
- recoveredRolledFiles = true;
+ rolloverOccurred = recoverRolledFiles(context, session, expectedChecksumValue, state.getTimestamp(), state.getPosition());
+ expectedRecoveryChecksum = null;
}
// initialize local variables from state object; this is done so that we can easily change the values throughout
@@ -479,57 +337,11 @@ public class TailFile extends AbstractProcessor {
final long startNanos = System.nanoTime();
// Check if file has rotated
- long fileLength = file.length();
- final long lastModified = file.lastModified();
- if (fileLength < position) {
- // File has rotated. It's possible that it rotated before we finished reading all of the data. As a result, we need
- // to check the last rolled-over file and see if it is longer than our position. If so, consume the data past our
- // marked position.
- try {
- final List<File> updatedRolledOverFiles = getRolledOffFiles(context, timestamp);
- getLogger().debug("Tailed file has rotated. Total number of rolled off files to check for un-consumed modifications: {}", new Object[] {updatedRolledOverFiles.size()});
-
- if (!updatedRolledOverFiles.isEmpty()) {
- final File lastRolledOver = updatedRolledOverFiles.get(updatedRolledOverFiles.size() - 1);
-
- // there is more data in the file that has not yet been consumed.
- if (lastRolledOver.length() > state.getPosition()) {
- getLogger().debug("Last rolled over file {} is larger than our last position; will consume data from it after offset {}", new Object[] {lastRolledOver, state.getPosition()});
-
- try (final FileInputStream fis = new FileInputStream(lastRolledOver)) {
- StreamUtils.skip(fis, state.getPosition());
-
- FlowFile flowFile = session.create();
- flowFile = session.importFrom(fis, flowFile);
- if (flowFile.getSize() == 0) {
- session.remove(flowFile);
- } else {
- flowFile = session.putAttribute(flowFile, "filename", lastRolledOver.getName());
-
- session.getProvenanceReporter().receive(flowFile, lastRolledOver.toURI().toString(), "FlowFile contains bytes " + state.getPosition() + " through " +
- lastRolledOver.length() + " of source file", TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos));
- session.transfer(flowFile, REL_SUCCESS);
- this.state = state = new TailFileState(context.getProperty(FILENAME).getValue(), null, null, 0L, lastRolledOver.lastModified() + 1L, null, state.getBuffer());
-
- // must ensure that we do session.commit() before persisting state in order to avoid data loss.
- session.commit();
- persistState(state, context.getProperty(STATE_FILE).getValue());
- }
- }
- } else {
- getLogger().debug("Last rolled over file {} is not larger than our last position; will not consume data from it", new Object[] {lastRolledOver});
- }
- }
- } catch (final IOException ioe) {
- getLogger().error("File being tailed was rolled over. However, was unable to determine which \"Rollover Files\" exist or read the last one due to {}. "
- + "It is possible that data at the end of the last file will be skipped as a result.", new Object[] {ioe});
- }
-
-
+ if (rolloverOccurred) {
// Since file has rotated, we close the reader, create a new one, and then reset our state.
try {
reader.close();
- getLogger().debug("Closed RandomAccessFile {}", new Object[] {reader, reader});
+ getLogger().debug("Closed FileChannel {}", new Object[] {reader, reader});
} catch (final IOException ioe) {
getLogger().warn("Failed to close reader for {} due to {}", new Object[] {file, ioe});
}
@@ -537,104 +349,85 @@ public class TailFile extends AbstractProcessor {
reader = createReader(file, 0L);
position = 0L;
checksum.reset();
- fileLength = file.length();
}
- // check if there is any data to consume by checking if file has grown or last modified timestamp has changed.
- boolean consumeData = false;
- if (fileLength > position) {
- consumeData = true;
- } else if (lastModified > timestamp) {
- // This can happen if file is truncated, or is replaced with the same amount of data as the old file.
- position = 0;
-
- try {
- reader.position(0L);
- } catch (final IOException ioe) {
- getLogger().error("Failed to seek to beginning of file due to {}", new Object[] {ioe});
- context.yield();
- return;
- }
-
- consumeData = true;
+ if (file.length() == position) {
+ // no data to consume so rather than continually running, yield to allow other processors to use the thread.
+ // In this case, the state should not have changed, and we will have created no FlowFiles, so we don't have to
+ // persist the state or commit the session; instead, just return here.
+ getLogger().debug("No data to consume; created no FlowFiles");
+ state = this.state = new TailFileState(context.getProperty(FILENAME).getValue(), file, reader, position, timestamp, checksum, state.getBuffer());
+ persistState(state, context);
+ context.yield();
+ return;
}
// If there is data to consume, read as much as we can.
final TailFileState currentState = state;
final Checksum chksum = checksum;
- if (consumeData) {
- // data has been written to file. Stream it to a new FlowFile.
- FlowFile flowFile = session.create();
-
- final FileChannel fileReader = reader;
- final LongHolder positionHolder = new LongHolder(position);
- flowFile = session.write(flowFile, new OutputStreamCallback() {
- @Override
- public void process(final OutputStream rawOut) throws IOException {
- try (final OutputStream out = new BufferedOutputStream(rawOut)) {
- positionHolder.set(readLines(fileReader, currentState.getBuffer(), out, chksum));
- }
- }
- });
+ // data has been written to file. Stream it to a new FlowFile.
+ FlowFile flowFile = session.create();
- // If there ended up being no data, just remove the FlowFile
- if (flowFile.getSize() == 0) {
- session.remove(flowFile);
- getLogger().debug("No data to consume; removed created FlowFile");
- } else {
- // determine filename for FlowFile by using <base filename of log file>.<initial offset>-<final offset>.<extension>
- final String tailFilename = file.getName();
- final String baseName = StringUtils.substringBeforeLast(tailFilename, ".");
- final String flowFileName;
- if (baseName.length() < tailFilename.length()) {
- flowFileName = baseName + "." + position + "-" + positionHolder.get() + "." + StringUtils.substringAfterLast(tailFilename, ".");
- } else {
- flowFileName = baseName + "." + position + "-" + positionHolder.get();
+ final FileChannel fileReader = reader;
+ final LongHolder positionHolder = new LongHolder(position);
+ flowFile = session.write(flowFile, new OutputStreamCallback() {
+ @Override
+ public void process(final OutputStream rawOut) throws IOException {
+ try (final OutputStream out = new BufferedOutputStream(rawOut)) {
+ positionHolder.set(readLines(fileReader, currentState.getBuffer(), out, chksum));
}
+ }
+ });
- final Map<String, String> attributes = new HashMap<>(2);
- attributes.put(CoreAttributes.FILENAME.key(), flowFileName);
- attributes.put(CoreAttributes.MIME_TYPE.key(), "text/plain");
- flowFile = session.putAllAttributes(flowFile, attributes);
-
- session.getProvenanceReporter().receive(flowFile, file.toURI().toString(), "FlowFile contains bytes " + position + " through " + positionHolder.get() + " of source file",
- TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos));
- session.transfer(flowFile, REL_SUCCESS);
- position = positionHolder.get();
- timestamp = lastModified;
- getLogger().debug("Created {} and routed to success", new Object[] {flowFile});
+ // If there ended up being no data, just remove the FlowFile
+ if (flowFile.getSize() == 0) {
+ session.remove(flowFile);
+ getLogger().debug("No data to consume; removed created FlowFile");
+ } else {
+ // determine filename for FlowFile by using <base filename of log file>.<initial offset>-<final offset>.<extension>
+ final String tailFilename = file.getName();
+ final String baseName = StringUtils.substringBeforeLast(tailFilename, ".");
+ final String flowFileName;
+ if (baseName.length() < tailFilename.length()) {
+ flowFileName = baseName + "." + position + "-" + positionHolder.get() + "." + StringUtils.substringAfterLast(tailFilename, ".");
+ } else {
+ flowFileName = baseName + "." + position + "-" + positionHolder.get();
}
+
+ final Map<String, String> attributes = new HashMap<>(2);
+ attributes.put(CoreAttributes.FILENAME.key(), flowFileName);
+ attributes.put(CoreAttributes.MIME_TYPE.key(), "text/plain");
+ flowFile = session.putAllAttributes(flowFile, attributes);
+
+ session.getProvenanceReporter().receive(flowFile, file.toURI().toString(), "FlowFile contains bytes " + position + " through " + positionHolder.get() + " of source file",
+ TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos));
+ session.transfer(flowFile, REL_SUCCESS);
+ position = positionHolder.get();
+
+ // Set timestamp to the latest of when the file was modified and the current timestamp stored in the state.
+ // We do this because when we read a file that has been rolled over, we set the state to 1 millisecond later than the last mod date
+ // in order to avoid ingesting that file again. If we then read from this file during the same second (or millisecond, depending on the
+ // operating system file last mod precision), then we could set the timestamp to a smaller value, which could result in reading in the
+ // rotated file a second time.
+ timestamp = Math.max(state.getTimestamp(), file.lastModified());
+ getLogger().debug("Created {} and routed to success", new Object[] {flowFile});
}
// Create a new state object to represent our current position, timestamp, etc.
final TailFileState updatedState = new TailFileState(context.getProperty(FILENAME).getValue(), file, reader, position, timestamp, checksum, state.getBuffer());
this.state = updatedState;
- if (!consumeData) {
- // no data to consume so rather than continually running, yield to allow other processors to use the thread.
- // In this case, the state should not have changed, and we will have created no FlowFiles, so we don't have to
- // persist the state or commit the session; instead, just return here.
- getLogger().debug("No data to consume; created no FlowFiles");
- context.yield();
- return;
- }
-
// We must commit session before persisting state in order to avoid data loss on restart
session.commit();
- final String stateFilename = context.getProperty(STATE_FILE).getValue();
- try {
- persistState(updatedState, stateFilename);
- } catch (final IOException e) {
- getLogger().warn("Failed to update state file {} due to {}; some data may be duplicated on restart of NiFi", new Object[] {stateFilename, e});
- }
+ persistState(updatedState, context);
}
/**
- * Read new lines from the given RandomAccessFile, copying it to the given Output Stream. The Checksum is used in order to later determine whether or not
+ * Read new lines from the given FileChannel, copying it to the given Output Stream. The Checksum is used in order to later determine whether or not
* data has been consumed.
*
- * @param reader The RandomAccessFile to read data from
+ * @param reader The FileChannel to read data from
* @param buffer the buffer to use for copying data
* @param out the OutputStream to copy the data to
* @param checksum the Checksum object to use in order to calculate checksum for recovery purposes
@@ -643,6 +436,8 @@ public class TailFile extends AbstractProcessor {
* @throws java.io.IOException if an I/O error occurs.
*/
private long readLines(final FileChannel reader, final ByteBuffer buffer, final OutputStream out, final Checksum checksum) throws IOException {
+ getLogger().debug("Reading lines starting at position {}", new Object[] {reader.position()});
+
try (final ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
long pos = reader.position();
long rePos = pos; // position to re-read
@@ -651,11 +446,12 @@ public class TailFile extends AbstractProcessor {
int linesRead = 0;
boolean seenCR = false;
buffer.clear();
+
while (((num = reader.read(buffer)) != -1)) {
buffer.flip();
for (int i = 0; i < num; i++) {
- byte ch = buffer.get();
+ byte ch = buffer.get(i);
switch (ch) {
case '\n':
@@ -697,9 +493,11 @@ public class TailFile extends AbstractProcessor {
pos = reader.position();
}
- getLogger().debug("Read {} lines; repositioning reader from {} to {}", new Object[] {linesRead, pos, rePos});
- reader.position(rePos);
- buffer.clear();
+ if (rePos < reader.position()) {
+ getLogger().debug("Read {} lines; repositioning reader from {} to {}", new Object[] {linesRead, pos, rePos});
+ reader.position(rePos); // Ensure we can re-read if necessary
+ }
+
return rePos;
}
}
@@ -773,25 +571,239 @@ public class TailFile extends AbstractProcessor {
}
+
+ private void persistState(final TailFileState state, final ProcessContext context) {
+ final String stateFilename = context.getProperty(STATE_FILE).getValue();
+ try {
+ persistState(state, stateFilename);
+ } catch (final IOException e) {
+ getLogger().warn("Failed to update state file {} due to {}; some data may be duplicated on restart of NiFi", new Object[] {stateFilename, e});
+ }
+ }
+
+ private void persistState(final TailFileState state, final String stateFilename) throws IOException {
+ getLogger().debug("Persisting state {} to {}", new Object[] {state, stateFilename});
+
+ final File stateFile = new File(stateFilename);
+ File directory = stateFile.getParentFile();
+ if (directory != null && !directory.exists() && !directory.mkdirs()) {
+ getLogger().warn("Failed to persist state to {} because the parent directory does not exist and could not be created. This may result in data being duplicated upon restart of NiFi");
+ return;
+ }
+ try (final FileOutputStream fos = new FileOutputStream(stateFile);
+ final DataOutputStream dos = new DataOutputStream(fos)) {
+
+ dos.writeInt(0); // version
+ dos.writeUTF(state.getFilename());
+ dos.writeLong(state.getPosition());
+ dos.writeLong(state.getTimestamp());
+ if (state.getChecksum() == null) {
+ dos.writeBoolean(false);
+ } else {
+ dos.writeBoolean(true);
+ dos.writeLong(state.getChecksum().getValue());
+ }
+ }
+ }
+
+ private FileChannel createReader(final File file, final long position) {
+ final FileChannel reader;
+
+ try {
+ reader = FileChannel.open(file.toPath(), StandardOpenOption.READ);
+ } catch (final IOException ioe) {
+ getLogger().warn("Unable to open file {}; will attempt to access file again after the configured Yield Duration has elapsed: {}", new Object[] {file, ioe});
+ return null;
+ }
+
+ getLogger().debug("Created FileChannel {} for {}", new Object[] {reader, file});
+
+ try {
+ reader.position(position);
+ } catch (final IOException ioe) {
+ getLogger().error("Failed to read from {} due to {}", new Object[] {file, ioe});
+
+ try {
+ reader.close();
+ getLogger().debug("Closed FileChannel {}", new Object[] {reader});
+ } catch (final IOException ioe2) {
+ }
+
+ return null;
+ }
+
+ return reader;
+ }
+
+ // for testing purposes
+ TailFileState getState() {
+ return state;
+ }
+
+
+ /**
+ * Finds any files that have rolled over and have not yet been ingested by this Processor. Each of these files that is found will be
+ * ingested as its own FlowFile. If a file is found that has been partially ingested, the rest of the file will be ingested as a
+ * single FlowFile but the data that already has been ingested will not be ingested again.
+ *
+ * @param context the ProcessContext to use in order to obtain Processor configuration.
+ * @param session the ProcessSession to use in order to interact with FlowFile creation and content.
+ * @param expectedChecksum the checksum value that is expected for the oldest file from offset 0 through <position>.
+ * @param timestamp the latest Last Modified Timestamp that has been consumed. Any data that was written before this data will not be ingested.
+ * @param position the byte offset in the file being tailed, where tailing last left off.
+ *
+ * @return <code>true</code> if the file being tailed has rolled over, <code>false</code> otherwise
+ */
+ private boolean recoverRolledFiles(final ProcessContext context, final ProcessSession session, final Long expectedChecksum, final long timestamp, final long position) {
+ try {
+ // Find all files that match our rollover pattern, if any, and order them based on their timestamp and filename.
+ // Ignore any file that has a timestamp earlier than the state that we have persisted. If we were reading from
+ // a file when we stopped running, then that file that we were reading from should be the first file in this list,
+ // assuming that the file still exists on the file system.
+ final List<File> rolledOffFiles = getRolledOffFiles(context, timestamp);
+ return recoverRolledFiles(context, session, rolledOffFiles, expectedChecksum, timestamp, position);
+ } catch (final IOException e) {
+ getLogger().error("Failed to recover files that have rolled over due to {}", new Object[] {e});
+ return false;
+ }
+ }
+
+ /**
+ * Finds any files that have rolled over and have not yet been ingested by this Processor. Each of these files that is found will be
+ * ingested as its own FlowFile. If a file is found that has been partially ingested, the rest of the file will be ingested as a
+ * single FlowFile but the data that already has been ingested will not be ingested again.
+ *
+ * @param context the ProcessContext to use in order to obtain Processor configuration.
+ * @param session the ProcessSession to use in order to interact with FlowFile creation and content.
+ * @param expectedChecksum the checksum value that is expected for the oldest file from offset 0 through <position>.
+ * @param timestamp the latest Last Modfiied Timestamp that has been consumed. Any data that was written before this data will not be ingested.
+ * @param position the byte offset in the file being tailed, where tailing last left off.
+ *
+ * @return <code>true</code> if the file being tailed has rolled over, false otherwise
+ */
+ private boolean recoverRolledFiles(final ProcessContext context, final ProcessSession session, final List<File> rolledOffFiles, final Long expectedChecksum,
+ final long timestamp, final long position) {
+ try {
+ getLogger().debug("Recovering Rolled Off Files; total number of files rolled off = {}", new Object[] {rolledOffFiles.size()});
+
+ // For first file that we find, it may or may not be the file that we were last reading from.
+ // As a result, we have to read up to the position we stored, while calculating the checksum. If the checksums match,
+ // then we know we've already processed this file. If the checksums do not match, then we have not
+ // processed this file and we need to seek back to position 0 and ingest the entire file.
+ // For all other files that have been rolled over, we need to just ingest the entire file.
+ boolean rolloverOccurred = !rolledOffFiles.isEmpty();
+ if (rolloverOccurred && expectedChecksum != null && rolledOffFiles.get(0).length() >= position) {
+ final File firstFile = rolledOffFiles.get(0);
+
+ final long startNanos = System.nanoTime();
+ if (position > 0) {
+ try (final InputStream fis = new FileInputStream(firstFile);
+ final CheckedInputStream in = new CheckedInputStream(fis, new CRC32())) {
+ StreamUtils.copy(in, new NullOutputStream(), position);
+
+ final long checksumResult = in.getChecksum().getValue();
+ if (checksumResult == expectedChecksum) {
+ getLogger().debug("Checksum for {} matched expected checksum. Will skip first {} bytes", new Object[] {firstFile, position});
+
+ // This is the same file that we were reading when we shutdown. Start reading from this point on.
+ rolledOffFiles.remove(0);
+ FlowFile flowFile = session.create();
+ flowFile = session.importFrom(in, flowFile);
+ if (flowFile.getSize() == 0L) {
+ session.remove(flowFile);
+ // use a timestamp of lastModified() + 1 so that we do not ingest this file again.
+ cleanup();
+ state = new TailFileState(context.getProperty(FILENAME).getValue(), null, null, 0L, firstFile.lastModified() + 1L, null, state.getBuffer());
+ } else {
+ flowFile = session.putAttribute(flowFile, "filename", firstFile.getName());
+
+ session.getProvenanceReporter().receive(flowFile, firstFile.toURI().toString(), "FlowFile contains bytes 0 through " + position + " of source file",
+ TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos));
+ session.transfer(flowFile, REL_SUCCESS);
+ getLogger().debug("Created {} from rolled over file {} and routed to success", new Object[] {flowFile, firstFile});
+
+ // use a timestamp of lastModified() + 1 so that we do not ingest this file again.
+ cleanup();
+ state = new TailFileState(context.getProperty(FILENAME).getValue(), null, null, 0L, firstFile.lastModified() + 1L, null, state.getBuffer());
+
+ // must ensure that we do session.commit() before persisting state in order to avoid data loss.
+ session.commit();
+ persistState(state, context.getProperty(STATE_FILE).getValue());
+ }
+ } else {
+ getLogger().debug("Checksum for {} did not match expected checksum. Checksum for file was {} but expected {}. Will consume entire file",
+ new Object[] {firstFile, checksumResult, expectedChecksum});
+ }
+ }
+ }
+ }
+
+ // For each file that we found that matches our Rollover Pattern, and has a last modified date later than the timestamp
+ // that we recovered from the state file, we need to consume the entire file. The only exception to this is the file that
+ // we were reading when we last stopped, as it may already have been partially consumed. That is taken care of in the
+ // above block of code.
+ for (final File file : rolledOffFiles) {
+ state = consumeFileFully(file, context, session, state);
+ }
+
+ return rolloverOccurred;
+ } catch (final IOException e) {
+ getLogger().error("Failed to recover files that have rolled over due to {}", new Object[] {e});
+ return false;
+ }
+ }
+
+
/**
- * A simple Java class to hold information about our state so that we can maintain this state across multiple invocations of the Processor.
+ * Creates a new FlowFile that contains the entire contents of the given file and transfers that FlowFile to success. This method
+ * will commit the given session and emit an appropriate Provenance Event.
+ *
+ * @param file the file to ingest
+ * @param context the ProcessContext
+ * @param session the ProcessSession
+ * @param state the current state
*
- * We use a FileChannel to read from the file, rather than a BufferedInputStream, etc. because we want to be able to read any amount of data
- * and then reposition the reader if we need to, as a result of a line not being terminated (i.e., no new-line).
+ * @return the new, updated state that reflects that the given file has been ingested.
+ */
+ private TailFileState consumeFileFully(final File file, final ProcessContext context, final ProcessSession session, TailFileState state) {
+ FlowFile flowFile = session.create();
+ flowFile = session.importFrom(file.toPath(), true, flowFile);
+ if (flowFile.getSize() == 0L) {
+ session.remove(flowFile);
+ } else {
+ flowFile = session.putAttribute(flowFile, "filename", file.getName());
+ session.getProvenanceReporter().receive(flowFile, file.toURI().toString());
+ session.transfer(flowFile, REL_SUCCESS);
+ getLogger().debug("Created {} from {} and routed to success", new Object[] {flowFile, file});
+
+ // use a timestamp of lastModified() + 1 so that we do not ingest this file again.
+ cleanup();
+ state = new TailFileState(context.getProperty(FILENAME).getValue(), null, null, 0L, file.lastModified() + 1L, null, state.getBuffer());
+
+ // must ensure that we do session.commit() before persisting state in order to avoid data loss.
+ session.commit();
+ persistState(state, context);
+ }
+
+ return state;
+ }
+
+ /**
+ * A simple Java class to hold information about our state so that we can maintain this state across multiple invocations of the Processor
*/
static class TailFileState {
private final String filename; // hold onto filename and not just File because we want to match that against the user-defined filename to recover from
private final File file;
- private final FileChannel fileChannel;
+ private final FileChannel reader;
private final long position;
private final long timestamp;
private final Checksum checksum;
private final ByteBuffer buffer;
- public TailFileState(final String filename, final File file, final FileChannel fileChannel, final long position, final long timestamp, final Checksum checksum, final ByteBuffer buffer) {
+ public TailFileState(final String filename, final File file, final FileChannel reader, final long position, final long timestamp, final Checksum checksum, final ByteBuffer buffer) {
this.filename = filename;
this.file = file;
- this.fileChannel = fileChannel;
+ this.reader = reader;
this.position = position;
this.timestamp = timestamp; // many operating systems will use only second-level precision for last-modified times so cut off milliseconds
this.checksum = checksum;
@@ -807,7 +819,7 @@ public class TailFile extends AbstractProcessor {
}
public FileChannel getReader() {
- return fileChannel;
+ return reader;
}
public long getPosition() {
http://git-wip-us.apache.org/repos/asf/nifi/blob/2516b1da/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTailFile.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTailFile.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTailFile.java
index 1445d88..e3d74c4 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTailFile.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTailFile.java
@@ -95,12 +95,14 @@ public class TestTailFile {
runner.run();
runner.assertAllFlowFilesTransferred(TailFile.REL_SUCCESS, 1);
runner.getFlowFilesForRelationship(TailFile.REL_SUCCESS).get(0).assertContentEquals("hello\n");
+ System.out.println("Ingested 6 bytes");
runner.clearTransferState();
// roll over the file
raf.close();
file.renameTo(new File(file.getParentFile(), file.getName() + ".previous"));
raf = new RandomAccessFile(file, "rw");
+ System.out.println("Rolled over file to " + file.getName() + ".previous");
// truncate file
raf.setLength(0L);
@@ -111,6 +113,7 @@ public class TestTailFile {
Thread.sleep(1000L); // we need to wait at least one second because of the granularity of timestamps on many file systems.
raf.write("HELLO\n".getBytes());
+ System.out.println("Wrote out 6 bytes to tailed file");
runner.run();
runner.assertAllFlowFilesTransferred(TailFile.REL_SUCCESS, 1);
runner.getFlowFilesForRelationship(TailFile.REL_SUCCESS).get(0).assertContentEquals("HELLO\n");
@@ -119,6 +122,7 @@ public class TestTailFile {
@Test
public void testConsumeAfterTruncationStartAtCurrentTime() throws IOException, InterruptedException {
runner.setProperty(TailFile.START_POSITION, TailFile.START_CURRENT_TIME.getValue());
+ runner.setProperty(TailFile.ROLLING_FILENAME_PATTERN, "log.txt*");
runner.run();
runner.assertAllFlowFilesTransferred(TailFile.REL_SUCCESS, 0);
@@ -129,7 +133,10 @@ public class TestTailFile {
runner.clearTransferState();
// truncate and then write same number of bytes
- raf.setLength(0L);
+ raf.close();
+ assertTrue(file.renameTo(new File("target/log.txt.1")));
+ raf = new RandomAccessFile(file, "rw");
+
runner.run();
runner.assertAllFlowFilesTransferred(TailFile.REL_SUCCESS, 0);
@@ -157,7 +164,7 @@ public class TestTailFile {
runner.setProperty(TailFile.START_POSITION, TailFile.START_CURRENT_TIME.getValue());
raf.write("hello world\n".getBytes());
- Thread.sleep(1000);
+ Thread.sleep(1000L);
runner.run(100);
runner.assertAllFlowFilesTransferred(TailFile.REL_SUCCESS, 0);
}
@@ -225,7 +232,7 @@ public class TestTailFile {
public void testRemainderOfFileRecoveredIfRolledOverWhileRunning() throws IOException {
// this mimics the case when we are reading a log file that rolls over while processor is running.
runner.setProperty(TailFile.ROLLING_FILENAME_PATTERN, "log*.txt");
- runner.run(1, false, false);
+ runner.run(1, false, true);
runner.assertAllFlowFilesTransferred(TailFile.REL_SUCCESS, 0);
raf.write("hello\n".getBytes());
@@ -236,13 +243,11 @@ public class TestTailFile {
raf.write("world".getBytes());
raf.close();
-
- processor.cleanup(); // Need to do this for Windows because otherwise we cannot rename the file because we have the file open still in the same process.
- assertTrue(file.renameTo(new File("target/log1.txt")));
+ file.renameTo(new File("target/log1.txt"));
raf = new RandomAccessFile(new File("target/log.txt"), "rw");
raf.write("1\n".getBytes());
- runner.run(1, false, false);
+ runner.run(1, true, false);
runner.assertAllFlowFilesTransferred(TailFile.REL_SUCCESS, 2);
runner.getFlowFilesForRelationship(TailFile.REL_SUCCESS).get(0).assertContentEquals("world");
@@ -258,7 +263,7 @@ public class TestTailFile {
// this mimics the case when we are reading a log file that rolls over while processor is running.
runner.setProperty(TailFile.ROLLING_FILENAME_PATTERN, "log.*");
- runner.run(1, false, false);
+ runner.run(1, false, true);
runner.assertAllFlowFilesTransferred(TailFile.REL_SUCCESS, 0);
raf.write("hello\n".getBytes());
@@ -279,7 +284,7 @@ public class TestTailFile {
raf = new RandomAccessFile(new File("target/log.txt"), "rw");
raf.write("1\n".getBytes());
- runner.run(1, false, false);
+ runner.run(1);
runner.assertAllFlowFilesTransferred(TailFile.REL_SUCCESS, 2);
runner.getFlowFilesForRelationship(TailFile.REL_SUCCESS).get(0).assertContentEquals("world");
@@ -291,7 +296,7 @@ public class TestTailFile {
public void testMultipleRolloversAfterHavingReadAllData() throws IOException, InterruptedException {
// this mimics the case when we are reading a log file that rolls over while processor is running.
runner.setProperty(TailFile.ROLLING_FILENAME_PATTERN, "log.*");
- runner.run(1, false, false);
+ runner.run(1, false, true);
runner.assertAllFlowFilesTransferred(TailFile.REL_SUCCESS, 0);
raf.write("hello\n".getBytes());
@@ -312,8 +317,52 @@ public class TestTailFile {
// write to a new file.
file = new File("target/log.txt");
raf = new RandomAccessFile(file, "rw");
+ raf.write("abc\n".getBytes());
+
+ // rename file to log.1
+ raf.close();
+ file.renameTo(new File("target/log.1"));
+
+ // write to a new file.
+ file = new File("target/log.txt");
+ raf = new RandomAccessFile(file, "rw");
+ raf.write("1\n".getBytes());
+ raf.close();
+
+ runner.run(1);
+
+ runner.assertAllFlowFilesTransferred(TailFile.REL_SUCCESS, 3);
+ runner.getFlowFilesForRelationship(TailFile.REL_SUCCESS).get(0).assertContentEquals("world");
+ runner.getFlowFilesForRelationship(TailFile.REL_SUCCESS).get(1).assertContentEquals("abc\n");
+ runner.getFlowFilesForRelationship(TailFile.REL_SUCCESS).get(2).assertContentEquals("1\n");
+ }
+
+
+ @Test
+ public void testMultipleRolloversAfterHavingReadAllDataWhileStillRunning() throws IOException, InterruptedException {
+ // this mimics the case when we are reading a log file that rolls over while processor is running.
+ runner.setProperty(TailFile.ROLLING_FILENAME_PATTERN, "log.*");
+ runner.run(1, false, true);
+ runner.assertAllFlowFilesTransferred(TailFile.REL_SUCCESS, 0);
+
+ raf.write("hello\n".getBytes());
+ runner.run(1, false, false);
+ runner.assertAllFlowFilesTransferred(TailFile.REL_SUCCESS, 1);
+ runner.getFlowFilesForRelationship(TailFile.REL_SUCCESS).get(0).assertContentEquals("hello\n");
+ runner.clearTransferState();
+
+ raf.write("world".getBytes());
+ runner.run(1, false, false); // ensure that we've read 'world' but not consumed it into a flowfile.
Thread.sleep(1000L);
+
+ // rename file to log.2
+ raf.close();
+ file.renameTo(new File("target/log.2"));
+
+ // write to a new file.
+ file = new File("target/log.txt");
+ raf = new RandomAccessFile(file, "rw");
raf.write("abc\n".getBytes());
// rename file to log.1
@@ -326,7 +375,7 @@ public class TestTailFile {
raf.write("1\n".getBytes());
raf.close();
- runner.run(1);
+ runner.run(1, true, false); // perform shutdown but do not perform initialization because last iteration didn't shutdown.
runner.assertAllFlowFilesTransferred(TailFile.REL_SUCCESS, 3);
runner.getFlowFilesForRelationship(TailFile.REL_SUCCESS).get(0).assertContentEquals("world");
@@ -336,10 +385,58 @@ public class TestTailFile {
@Test
+ public void testMultipleRolloversWithLongerFileLength() throws IOException, InterruptedException {
+ // this mimics the case when we are reading a log file that rolls over while processor is running.
+ runner.setProperty(TailFile.ROLLING_FILENAME_PATTERN, "log.*");
+ runner.run(1, false, true);
+ runner.assertAllFlowFilesTransferred(TailFile.REL_SUCCESS, 0);
+
+ raf.write("hello\n".getBytes());
+ runner.run(1, false, false);
+ runner.assertAllFlowFilesTransferred(TailFile.REL_SUCCESS, 1);
+ runner.getFlowFilesForRelationship(TailFile.REL_SUCCESS).get(0).assertContentEquals("hello\n");
+ runner.clearTransferState();
+
+ raf.write("world".getBytes());
+
+ // rename file to log.2
+ raf.close();
+ file.renameTo(new File("target/log.2"));
+
+ Thread.sleep(1200L);
+
+ // write to a new file.
+ file = new File("target/log.txt");
+ raf = new RandomAccessFile(file, "rw");
+ raf.write("abc\n".getBytes());
+
+ // rename file to log.1
+ raf.close();
+ file.renameTo(new File("target/log.1"));
+ Thread.sleep(1200L);
+
+ // write to a new file.
+ file = new File("target/log.txt");
+ raf = new RandomAccessFile(file, "rw");
+ raf.write("This is a longer line than the other files had.\n".getBytes());
+ raf.close();
+
+ runner.run(1, true, false); // perform shutdown but do not perform initialization because last iteration didn't shutdown.
+
+ runner.assertAllFlowFilesTransferred(TailFile.REL_SUCCESS, 3);
+ runner.getFlowFilesForRelationship(TailFile.REL_SUCCESS).get(0).assertContentEquals("world");
+ runner.getFlowFilesForRelationship(TailFile.REL_SUCCESS).get(1).assertContentEquals("abc\n");
+ runner.getFlowFilesForRelationship(TailFile.REL_SUCCESS).get(2).assertContentEquals("This is a longer line than the other files had.\n");
+ }
+
+
+ @Test
public void testConsumeWhenNewLineFound() throws IOException, InterruptedException {
runner.run();
runner.assertAllFlowFilesTransferred(TailFile.REL_SUCCESS, 0);
+ Thread.sleep(1100L);
+
raf.write("Hello, World".getBytes());
runner.run();
runner.assertAllFlowFilesTransferred(TailFile.REL_SUCCESS, 0);
@@ -377,7 +474,39 @@ public class TestTailFile {
runner.run();
runner.assertAllFlowFilesTransferred(TailFile.REL_SUCCESS, 1);
runner.getFlowFilesForRelationship(TailFile.REL_SUCCESS).get(0).assertContentEquals("return\r\r\n");
+ }
+
+ @Test
+ public void testRolloverAndUpdateAtSameTime() throws IOException {
+ runner.setProperty(TailFile.ROLLING_FILENAME_PATTERN, "log.*");
+ // write out some data and ingest it.
+ raf.write("hello there\n".getBytes());
+ runner.run();
+ runner.assertAllFlowFilesTransferred(TailFile.REL_SUCCESS, 1);
+ runner.clearTransferState();
+
+ // roll the file over and write data to the new log.txt file.
+ raf.write("another".getBytes());
+ raf.close();
+ file.renameTo(new File("target/log.1"));
+ raf = new RandomAccessFile(file, "rw");
+ raf.write("new file\n".getBytes());
+
+ // Run the processor. We should get 2 files because we should get the rest of what was
+ // written to log.txt before it rolled, and then we should get some data from the new log.txt.
+ runner.run(1, false, true);
+ runner.assertAllFlowFilesTransferred(TailFile.REL_SUCCESS, 2);
+ runner.getFlowFilesForRelationship(TailFile.REL_SUCCESS).get(0).assertContentEquals("another");
+
+ // If we run again, we should get nothing.
+ // We did have an issue where we were recognizing the previously rolled over file again because the timestamps
+ // were still the same (second-level precision on many file systems). As a result, we verified the checksum of the
+ // already-rolled file against the checksum of the new file and they didn't match, so we ingested the entire rolled
+ // file as well as the new file again. Instead, we should ingest nothing!
+ runner.clearTransferState();
+ runner.run(1, true, false);
+ runner.assertAllFlowFilesTransferred(TailFile.REL_SUCCESS, 0);
}
}
[16/50] [abbrv] nifi git commit: NIFI-1176: Use a smaller internal
blocking queue size of only 5000 messages;
do not create a new queue every time the processor is scheduled to run
Posted by mc...@apache.org.
NIFI-1176: Use a smaller internal blocking queue size of only 5000 messages; do not create a new queue every time the processor is scheduled to run
Signed-off-by: Aldrin Piri <al...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/180ea1ba
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/180ea1ba
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/180ea1ba
Branch: refs/heads/NIFI-655
Commit: 180ea1ba222a5890541dfbe046f2876695e69239
Parents: 93be753
Author: Mark Payne <ma...@hotmail.com>
Authored: Mon Nov 16 15:50:14 2015 -0500
Committer: Aldrin Piri <al...@apache.org>
Committed: Wed Nov 18 10:51:22 2015 -0500
----------------------------------------------------------------------
.../main/java/org/apache/nifi/processors/twitter/GetTwitter.java | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/180ea1ba/nifi-nar-bundles/nifi-social-media-bundle/nifi-twitter-processors/src/main/java/org/apache/nifi/processors/twitter/GetTwitter.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-social-media-bundle/nifi-twitter-processors/src/main/java/org/apache/nifi/processors/twitter/GetTwitter.java b/nifi-nar-bundles/nifi-social-media-bundle/nifi-twitter-processors/src/main/java/org/apache/nifi/processors/twitter/GetTwitter.java
index e41b583..e099449 100644
--- a/nifi-nar-bundles/nifi-social-media-bundle/nifi-twitter-processors/src/main/java/org/apache/nifi/processors/twitter/GetTwitter.java
+++ b/nifi-nar-bundles/nifi-social-media-bundle/nifi-twitter-processors/src/main/java/org/apache/nifi/processors/twitter/GetTwitter.java
@@ -156,7 +156,7 @@ public class GetTwitter extends AbstractProcessor {
private final BlockingQueue<Event> eventQueue = new LinkedBlockingQueue<>(1000);
private volatile Client client;
- private volatile BlockingQueue<String> messageQueue = new LinkedBlockingQueue<>(10000);
+ private volatile BlockingQueue<String> messageQueue = new LinkedBlockingQueue<>(5000);
@Override
protected void init(final ProcessorInitializationContext context) {
@@ -222,8 +222,6 @@ public class GetTwitter extends AbstractProcessor {
@OnScheduled
public void onScheduled(final ProcessContext context) throws MalformedURLException {
- messageQueue = new LinkedBlockingQueue<>(100000);
-
final String endpointName = context.getProperty(ENDPOINT).getValue();
final Authentication oauth = new OAuth1(context.getProperty(CONSUMER_KEY).getValue(),
context.getProperty(CONSUMER_SECRET).getValue(),
[18/50] [abbrv] nifi git commit: NIFI-1108: Updated processors to
include the @InputRequirement annotation
Posted by mc...@apache.org.
NIFI-1108: Updated processors to include the @InputRequirement annotation
Signed-off-by: Aldrin Piri <al...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/911e1c54
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/911e1c54
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/911e1c54
Branch: refs/heads/NIFI-655
Commit: 911e1c54126bdff53b0b853274798e4292c63540
Parents: d88b6cb
Author: Mark Payne <ma...@hotmail.com>
Authored: Fri Nov 13 10:45:19 2015 -0500
Committer: Aldrin Piri <al...@apache.org>
Committed: Wed Nov 18 13:47:59 2015 -0500
----------------------------------------------------------------------
.../main/java/org/apache/nifi/processors/standard/RouteText.java | 3 +++
.../main/java/org/apache/nifi/processors/standard/TailFile.java | 3 +++
2 files changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/911e1c54/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/RouteText.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/RouteText.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/RouteText.java
index 5ba2a98..c412126 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/RouteText.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/RouteText.java
@@ -39,8 +39,10 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.annotation.behavior.DynamicProperty;
import org.apache.nifi.annotation.behavior.DynamicRelationship;
import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
import org.apache.nifi.annotation.behavior.SideEffectFree;
import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.annotation.lifecycle.OnScheduled;
@@ -66,6 +68,7 @@ import org.apache.nifi.processors.standard.util.NLKBufferedReader;
@EventDriven
@SideEffectFree
@SupportsBatching
+@InputRequirement(Requirement.INPUT_REQUIRED)
@Tags({"attributes", "routing", "text", "regexp", "regex", "Regular Expression", "Expression Language"})
@CapabilityDescription("Routes textual data based on a set of user-defined rules. Each line in an incoming FlowFile is compared against the values specified by user-defined Properties. "
+ "The mechanism by which the text is compared to these user-defined properties is defined by the 'Matching Strategy'. The data is then routed according to these rules, routing "
http://git-wip-us.apache.org/repos/asf/nifi/blob/911e1c54/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TailFile.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TailFile.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TailFile.java
index 4d7e6f5..684e3cc 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TailFile.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TailFile.java
@@ -44,7 +44,9 @@ import java.util.zip.CheckedInputStream;
import java.util.zip.Checksum;
import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.annotation.behavior.InputRequirement;
import org.apache.nifi.annotation.behavior.TriggerSerially;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.annotation.lifecycle.OnScheduled;
@@ -71,6 +73,7 @@ import org.apache.nifi.util.LongHolder;
+ "new line is encountered (carriage return or new-line character or combination). If the file to tail is periodically \"rolled over\", as is generally the case "
+ "with log files, an optional Rolling Filename Pattern can be used to retrieve data from files that have rolled over, even if the rollover occurred while NiFi "
+ "was not running (provided that the data still exists upon restart of NiFi).")
+@InputRequirement(Requirement.INPUT_FORBIDDEN)
public class TailFile extends AbstractProcessor {
static final AllowableValue START_BEGINNING_OF_TIME = new AllowableValue("Beginning of Time", "Beginning of Time",
[25/50] [abbrv] nifi git commit: NIFI-1081 Adding option to
ExecuteStreamCommand to put output value to an attribute
Posted by mc...@apache.org.
NIFI-1081 Adding option to ExecuteStreamCommand to put output value to an attribute
Reviewed and amended (comments,whitespace,and some code readability (discussed in ticket)) by Tony Kurc (tkurc@apache.org)
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/1e5cc070
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/1e5cc070
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/1e5cc070
Branch: refs/heads/NIFI-655
Commit: 1e5cc070a3d29736beea9af0b2d684a9bdcfff8e
Parents: 9e2f6df
Author: Joseph Percivall <jo...@yahoo.com>
Authored: Wed Nov 18 21:16:12 2015 -0500
Committer: Tony Kurc <tr...@gmail.com>
Committed: Wed Nov 18 23:23:10 2015 -0500
----------------------------------------------------------------------
NOTICE | 3 +
nifi-assembly/NOTICE | 4 +
.../standard/ExecuteStreamCommand.java | 200 ++++++++++++----
.../SoftLimitBoundedByteArrayOutputStream.java | 99 ++++++++
.../standard/TestExecuteStreamCommand.java | 237 ++++++++++++++++++-
5 files changed, 488 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/1e5cc070/NOTICE
----------------------------------------------------------------------
diff --git a/NOTICE b/NOTICE
index 9686fba..f39a045 100644
--- a/NOTICE
+++ b/NOTICE
@@ -4,3 +4,6 @@ Copyright 2014-2015 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
+This product includes the following work from the Apache Hadoop project:
+
+BoundedByteArrayOutputStream.java adapted to SoftLimitBoundedByteArrayOutputStream.java
http://git-wip-us.apache.org/repos/asf/nifi/blob/1e5cc070/nifi-assembly/NOTICE
----------------------------------------------------------------------
diff --git a/nifi-assembly/NOTICE b/nifi-assembly/NOTICE
index 5e18035..6ec5c31 100644
--- a/nifi-assembly/NOTICE
+++ b/nifi-assembly/NOTICE
@@ -4,6 +4,10 @@ Copyright 2014-2015 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
+This product includes the following work from the Apache Hadoop project:
+
+BoundedByteArrayOutputStream.java which was adapted to SoftLimitBoundedByteArrayOutputStream.java
+
===========================================
Apache Software License v2
===========================================
http://git-wip-us.apache.org/repos/asf/nifi/blob/1e5cc070/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ExecuteStreamCommand.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ExecuteStreamCommand.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ExecuteStreamCommand.java
index 9bea6ba..38c8bd4 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ExecuteStreamCommand.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ExecuteStreamCommand.java
@@ -30,6 +30,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
@@ -59,6 +60,7 @@ import org.apache.nifi.processor.io.InputStreamCallback;
import org.apache.nifi.processor.io.OutputStreamCallback;
import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.processors.standard.util.ArgumentUtils;
+import org.apache.nifi.processors.standard.util.SoftLimitBoundedByteArrayOutputStream;
import org.apache.nifi.stream.io.BufferedInputStream;
import org.apache.nifi.stream.io.BufferedOutputStream;
import org.apache.nifi.stream.io.StreamUtils;
@@ -126,7 +128,7 @@ import org.apache.nifi.stream.io.StreamUtils;
@CapabilityDescription("Executes an external command on the contents of a flow file, and creates a new flow file with the results of the command.")
@DynamicProperty(name = "An environment variable name", value = "An environment variable value", description = "These environment variables are passed to the process spawned by this Processor")
@WritesAttributes({
- @WritesAttribute(attribute = "execution.command", description = "The name of the command executed to create the new FlowFile"),
+ @WritesAttribute(attribute = "execution.command", description = "The name of the command executed"),
@WritesAttribute(attribute = "execution.command.args", description = "The semi-colon delimited list of arguments"),
@WritesAttribute(attribute = "execution.status", description = "The exit status code returned from executing the command"),
@WritesAttribute(attribute = "execution.error", description = "Any error messages returned from executing the command")})
@@ -140,14 +142,10 @@ public class ExecuteStreamCommand extends AbstractProcessor {
.name("output stream")
.description("The destination path for the flow file created from the command's output")
.build();
- private static final Set<Relationship> RELATIONSHIPS;
+ private AtomicReference<Set<Relationship>> relationships = new AtomicReference<>();
- static {
- Set<Relationship> rels = new HashSet<>();
- rels.add(OUTPUT_STREAM_RELATIONSHIP);
- rels.add(ORIGINAL_RELATIONSHIP);
- RELATIONSHIPS = Collections.unmodifiableSet(rels);
- }
+ private final static Set<Relationship> OUTPUT_STREAM_RELATIONSHIP_SET;
+ private final static Set<Relationship> ATTRIBUTE_RELATIONSHIP_SET;
private static final Validator ATTRIBUTE_EXPRESSION_LANGUAGE_VALIDATOR = StandardValidators.createAttributeExpressionLanguageValidator(ResultType.STRING, true);
static final PropertyDescriptor EXECUTION_COMMAND = new PropertyDescriptor.Builder()
@@ -195,6 +193,22 @@ public class ExecuteStreamCommand extends AbstractProcessor {
.defaultValue("false")
.build();
+ static final PropertyDescriptor PUT_OUTPUT_IN_ATTRIBUTE = new PropertyDescriptor.Builder()
+ .name("Output Destination Attribute")
+ .description("If set, the output of the stream command will be put into an attribute of the original FlowFile instead of a separate "
+ + "FlowFile. There will no longer be a relationship for 'output stream'. The value of this property will be the key for the output attribute.")
+ .addValidator(StandardValidators.ATTRIBUTE_KEY_PROPERTY_NAME_VALIDATOR)
+ .build();
+
+ static final PropertyDescriptor PUT_ATTRIBUTE_MAX_LENGTH = new PropertyDescriptor.Builder()
+ .name("Max Attribute Length")
+ .description("If routing the output of the stream command to an attribute, the number of characters put to the attribute value "
+ + "will be at most this amount. This is important because attributes are held in memory and large attributes will quickly "
+ + "cause out of memory issues. If the output goes longer than this value, it will truncated to fit. Consider making this smaller if able.")
+ .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR)
+ .defaultValue("256")
+ .build();
+
private static final Validator characterValidator = new StandardValidators.StringLengthValidator(1, 1);
static final PropertyDescriptor ARG_DELIMITER = new PropertyDescriptor.Builder()
@@ -216,19 +230,44 @@ public class ExecuteStreamCommand extends AbstractProcessor {
props.add(IGNORE_STDIN);
props.add(WORKING_DIR);
props.add(ARG_DELIMITER);
+ props.add(PUT_OUTPUT_IN_ATTRIBUTE);
+ props.add(PUT_ATTRIBUTE_MAX_LENGTH);
PROPERTIES = Collections.unmodifiableList(props);
+
+
+ Set<Relationship> outputStreamRelationships = new HashSet<>();
+ outputStreamRelationships.add(OUTPUT_STREAM_RELATIONSHIP);
+ outputStreamRelationships.add(ORIGINAL_RELATIONSHIP);
+ OUTPUT_STREAM_RELATIONSHIP_SET = Collections.unmodifiableSet(outputStreamRelationships);
+
+ Set<Relationship> attributeRelationships = new HashSet<>();
+ attributeRelationships.add(ORIGINAL_RELATIONSHIP);
+ ATTRIBUTE_RELATIONSHIP_SET = Collections.unmodifiableSet(attributeRelationships);
}
private ProcessorLog logger;
@Override
public Set<Relationship> getRelationships() {
- return RELATIONSHIPS;
+ return relationships.get();
}
@Override
protected void init(ProcessorInitializationContext context) {
logger = getLogger();
+
+ relationships.set(OUTPUT_STREAM_RELATIONSHIP_SET);
+ }
+
+ @Override
+ public void onPropertyModified(final PropertyDescriptor descriptor, final String oldValue, final String newValue) {
+ if (descriptor.equals(PUT_OUTPUT_IN_ATTRIBUTE)) {
+ if (newValue != null) {
+ relationships.set(ATTRIBUTE_RELATIONSHIP_SET);
+ } else {
+ relationships.set(OUTPUT_STREAM_RELATIONSHIP_SET);
+ }
+ }
}
@Override
@@ -254,6 +293,10 @@ public class ExecuteStreamCommand extends AbstractProcessor {
}
final ArrayList<String> args = new ArrayList<>();
+ final boolean putToAttribute = context.getProperty(PUT_OUTPUT_IN_ATTRIBUTE).isSet();
+ final Integer attributeSize = context.getProperty(PUT_ATTRIBUTE_MAX_LENGTH).asInteger();
+ final String attributeName = context.getProperty(PUT_OUTPUT_IN_ATTRIBUTE).getValue();
+
final String executeCommand = context.getProperty(EXECUTION_COMMAND).evaluateAttributeExpressions(inputFlowFile).getValue();
args.add(executeCommand);
final String commandArguments = context.getProperty(EXECUTION_ARGUMENTS).getValue();
@@ -300,10 +343,17 @@ public class ExecuteStreamCommand extends AbstractProcessor {
final BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(pes))) {
int exitCode = -1;
final BufferedOutputStream bos = new BufferedOutputStream(pos);
- FlowFile outputStreamFlowFile = session.create(inputFlowFile);
- ProcessStreamWriterCallback callback = new ProcessStreamWriterCallback(ignoreStdin, bos, bis, logger, session, outputStreamFlowFile, process);
+ FlowFile outputFlowFile = putToAttribute ? inputFlowFile : session.create(inputFlowFile);
+
+ ProcessStreamWriterCallback callback = new ProcessStreamWriterCallback(ignoreStdin, bos, bis, logger,
+ attributeName, session, outputFlowFile, process,putToAttribute,attributeSize);
session.read(inputFlowFile, callback);
- outputStreamFlowFile = callback.outputStreamFlowFile;
+
+ outputFlowFile = callback.outputFlowFile;
+ if (putToAttribute) {
+ outputFlowFile = session.putAttribute(outputFlowFile, attributeName, new String(callback.outputBuffer, 0, callback.size));
+ }
+
exitCode = callback.exitCode;
logger.debug("Execution complete for command: {}. Exited with code: {}", new Object[]{executeCommand, exitCode});
@@ -321,21 +371,29 @@ public class ExecuteStreamCommand extends AbstractProcessor {
int length = strBldr.length() > 4000 ? 4000 : strBldr.length();
attributes.put("execution.error", strBldr.substring(0, length));
+ final Relationship outputFlowFileRelationship = putToAttribute ? ORIGINAL_RELATIONSHIP : OUTPUT_STREAM_RELATIONSHIP;
if (exitCode == 0) {
- logger.info("Transferring flow file {} to 'output stream'", new Object[]{outputStreamFlowFile});
+ logger.info("Transferring flow file {} to {}",
+ new Object[]{outputFlowFile,outputFlowFileRelationship.getName()});
} else {
- logger.error("Transferring flow file {} to 'output stream'. Executable command {} ended in an error: {}",
- new Object[]{outputStreamFlowFile, executeCommand, strBldr.toString()});
+ logger.error("Transferring flow file {} to {}. Executable command {} ended in an error: {}",
+ new Object[]{outputFlowFile,outputFlowFileRelationship.getName(), executeCommand, strBldr.toString()});
}
attributes.put("execution.status", Integer.toString(exitCode));
attributes.put("execution.command", executeCommand);
attributes.put("execution.command.args", commandArguments);
- outputStreamFlowFile = session.putAllAttributes(outputStreamFlowFile, attributes);
- session.transfer(outputStreamFlowFile, OUTPUT_STREAM_RELATIONSHIP);
- logger.info("Transferring flow file {} to original", new Object[]{inputFlowFile});
- inputFlowFile = session.putAllAttributes(inputFlowFile, attributes);
- session.transfer(inputFlowFile, ORIGINAL_RELATIONSHIP);
+ outputFlowFile = session.putAllAttributes(outputFlowFile, attributes);
+
+ // This transfer will transfer the FlowFile that received the stream out put to it's destined relationship.
+ // In the event the stream is put to the an attribute of the original, it will be transferred here.
+ session.transfer(outputFlowFile, outputFlowFileRelationship);
+
+ if (!putToAttribute) {
+ logger.info("Transferring flow file {} to original", new Object[]{inputFlowFile});
+ inputFlowFile = session.putAllAttributes(inputFlowFile, attributes);
+ session.transfer(inputFlowFile, ORIGINAL_RELATIONSHIP);
+ }
} catch (final IOException ex) {
// could not close Process related streams
@@ -348,59 +406,97 @@ public class ExecuteStreamCommand extends AbstractProcessor {
static class ProcessStreamWriterCallback implements InputStreamCallback {
final boolean ignoreStdin;
- final OutputStream stdInWritable;
- final InputStream stdOutReadable;
+ final OutputStream stdinWritable;
+ final InputStream stdoutReadable;
final ProcessorLog logger;
final ProcessSession session;
final Process process;
- FlowFile outputStreamFlowFile;
+ FlowFile outputFlowFile;
int exitCode;
+ final boolean putToAttribute;
+ final int attributeSize;
+ final String attributeName;
+
+ byte[] outputBuffer;
+ int size;
- public ProcessStreamWriterCallback(boolean ignoreStdin, OutputStream stdInWritable, InputStream stdOutReadable,
- ProcessorLog logger, ProcessSession session, FlowFile outputStreamFlowFile, Process process) {
+ public ProcessStreamWriterCallback(boolean ignoreStdin, OutputStream stdinWritable, InputStream stdoutReadable,ProcessorLog logger, String attributeName,
+ ProcessSession session, FlowFile outputFlowFile, Process process, boolean putToAttribute, int attributeSize) {
this.ignoreStdin = ignoreStdin;
- this.stdInWritable = stdInWritable;
- this.stdOutReadable = stdOutReadable;
+ this.stdinWritable = stdinWritable;
+ this.stdoutReadable = stdoutReadable;
this.logger = logger;
this.session = session;
- this.outputStreamFlowFile = outputStreamFlowFile;
+ this.outputFlowFile = outputFlowFile;
this.process = process;
+ this.putToAttribute = putToAttribute;
+ this.attributeSize = attributeSize;
+ this.attributeName = attributeName;
}
@Override
public void process(final InputStream incomingFlowFileIS) throws IOException {
- outputStreamFlowFile = session.write(outputStreamFlowFile, new OutputStreamCallback() {
+ if (putToAttribute) {
+ try (SoftLimitBoundedByteArrayOutputStream softLimitBoundedBAOS = new SoftLimitBoundedByteArrayOutputStream(attributeSize)) {
+ readStdoutReadable(ignoreStdin, stdinWritable, logger, incomingFlowFileIS);
+ final long longSize = StreamUtils.copy(stdoutReadable, softLimitBoundedBAOS);
+
+ // Because the outputstream has a cap that the copy doesn't know about, adjust
+ // the actual size
+ if (longSize > (long) attributeSize) { // Explicit cast for readability
+ size = attributeSize;
+ } else{
+ size = (int) longSize; // Note: safe cast, longSize is limited by attributeSize
+ }
+
+ outputBuffer = softLimitBoundedBAOS.getBuffer();
+ stdoutReadable.close();
- @Override
- public void process(OutputStream out) throws IOException {
-
- Thread writerThread = new Thread(new Runnable() {
-
- @Override
- public void run() {
- if (!ignoreStdin) {
- try {
- StreamUtils.copy(incomingFlowFileIS, stdInWritable);
- } catch (IOException e) {
- logger.error("Failed to write flow file to stdIn due to {}", new Object[]{e}, e);
- }
- }
- // MUST close the output stream to the stdIn so that whatever is reading knows
- // there is no more data
- IOUtils.closeQuietly(stdInWritable);
- }
- });
- writerThread.setDaemon(true);
- writerThread.start();
- StreamUtils.copy(stdOutReadable, out);
try {
exitCode = process.waitFor();
} catch (InterruptedException e) {
logger.warn("Command Execution Process was interrupted", e);
}
}
- });
+ } else {
+ outputFlowFile = session.write(outputFlowFile, new OutputStreamCallback() {
+ @Override
+ public void process(OutputStream out) throws IOException {
+
+ readStdoutReadable(ignoreStdin, stdinWritable, logger, incomingFlowFileIS);
+ StreamUtils.copy(stdoutReadable, out);
+ try {
+ exitCode = process.waitFor();
+ } catch (InterruptedException e) {
+ logger.warn("Command Execution Process was interrupted", e);
+ }
+ }
+ });
+ }
}
}
+ private static void readStdoutReadable(final boolean ignoreStdin, final OutputStream stdinWritable,
+ final ProcessorLog logger, final InputStream incomingFlowFileIS) throws IOException {
+ Thread writerThread = new Thread(new Runnable() {
+
+ @Override
+ public void run() {
+ if (!ignoreStdin) {
+ try {
+ StreamUtils.copy(incomingFlowFileIS, stdinWritable);
+ } catch (IOException e) {
+ // This is unlikely to occur, and isn't handled at the moment
+ // Bug captured in NIFI-1194
+ logger.error("Failed to write flow file to stdin due to {}", new Object[]{e}, e);
+ }
+ }
+ // MUST close the output stream to the stdin so that whatever is reading knows
+ // there is no more data.
+ IOUtils.closeQuietly(stdinWritable);
+ }
+ });
+ writerThread.setDaemon(true);
+ writerThread.start();
+ }
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/1e5cc070/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/SoftLimitBoundedByteArrayOutputStream.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/SoftLimitBoundedByteArrayOutputStream.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/SoftLimitBoundedByteArrayOutputStream.java
new file mode 100644
index 0000000..95e9a72
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/SoftLimitBoundedByteArrayOutputStream.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard.util;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+public class SoftLimitBoundedByteArrayOutputStream extends OutputStream {
+ /*
+ * This Bounded Array Output Stream (BAOS) allows the user to write to the output stream up to a specified limit.
+ * Higher than that limit the BAOS will silently return and not put more into the buffer. It also will not throw an error.
+ * This effectively truncates the stream for the user to fit into a bounded array.
+ */
+
+ private final byte[] buffer;
+ private int limit;
+ private int count;
+
+ public SoftLimitBoundedByteArrayOutputStream(int capacity) {
+ this(capacity, capacity);
+ }
+
+ public SoftLimitBoundedByteArrayOutputStream(int capacity, int limit) {
+ if ((capacity < limit) || (capacity | limit) < 0) {
+ throw new IllegalArgumentException("Invalid capacity/limit");
+ }
+ this.buffer = new byte[capacity];
+ this.limit = limit;
+ this.count = 0;
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ if (count >= limit) {
+ return;
+ }
+ buffer[count++] = (byte) b;
+ }
+
+ @Override
+ public void write(byte b[], int off, int len) throws IOException {
+ if ((off < 0) || (off > b.length) || (len < 0) || ((off + len) > b.length)
+ || ((off + len) < 0)) {
+ throw new IndexOutOfBoundsException();
+ } else if (len == 0) {
+ return;
+ }
+
+ if (count + len > limit) {
+ len = limit-count;
+ if(len == 0){
+ return;
+ }
+ }
+
+ System.arraycopy(b, off, buffer, count, len);
+ count += len;
+ }
+
+ public void reset(int newlim) {
+ if (newlim > buffer.length) {
+ throw new IndexOutOfBoundsException("Limit exceeds buffer size");
+ }
+ this.limit = newlim;
+ this.count = 0;
+ }
+
+ public void reset() {
+ this.limit = buffer.length;
+ this.count = 0;
+ }
+
+ public int getLimit() {
+ return limit;
+ }
+
+ public byte[] getBuffer() {
+ return buffer;
+ }
+
+ public int size() {
+ return count;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/1e5cc070/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestExecuteStreamCommand.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestExecuteStreamCommand.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestExecuteStreamCommand.java
index 0f13ba2..44576d4 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestExecuteStreamCommand.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestExecuteStreamCommand.java
@@ -38,9 +38,6 @@ import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
-/**
- *
- */
public class TestExecuteStreamCommand {
@BeforeClass
public static void init() {
@@ -232,6 +229,237 @@ public class TestExecuteStreamCommand {
}
@Test
+ public void testSmallEchoPutToAttribute() throws Exception {
+ File dummy = new File("src/test/resources/hello.txt");
+ assertTrue(dummy.exists());
+ final TestRunner controller = TestRunners.newTestRunner(ExecuteStreamCommand.class);
+ controller.setValidateExpressionUsage(false);
+ controller.enqueue("".getBytes());
+
+ if(isWindows()) {
+ controller.setProperty(ExecuteStreamCommand.EXECUTION_COMMAND, "cmd.exe");
+ controller.setProperty(ExecuteStreamCommand.EXECUTION_ARGUMENTS, "/c;echo Hello");
+ controller.setProperty(ExecuteStreamCommand.ARG_DELIMITER, ";");
+ } else{
+ controller.setProperty(ExecuteStreamCommand.EXECUTION_COMMAND, "echo");
+ controller.setProperty(ExecuteStreamCommand.EXECUTION_ARGUMENTS, "Hello");
+ }
+ controller.setProperty(ExecuteStreamCommand.IGNORE_STDIN, "true");
+ controller.setProperty(ExecuteStreamCommand.PUT_OUTPUT_IN_ATTRIBUTE, "executeStreamCommand.output");
+
+ controller.run(1);
+ controller.assertTransferCount(ExecuteStreamCommand.ORIGINAL_RELATIONSHIP, 1);
+ controller.assertTransferCount(ExecuteStreamCommand.OUTPUT_STREAM_RELATIONSHIP, 0);
+
+ List<MockFlowFile> flowFiles = controller.getFlowFilesForRelationship(ExecuteStreamCommand.ORIGINAL_RELATIONSHIP);
+ MockFlowFile outputFlowFile = flowFiles.get(0);
+ outputFlowFile.assertContentEquals("");
+ String ouput = outputFlowFile.getAttribute("executeStreamCommand.output");
+ assertTrue(ouput.startsWith("Hello"));
+ assertEquals("0", outputFlowFile.getAttribute("execution.status"));
+ assertEquals(isWindows() ? "cmd.exe" : "echo", outputFlowFile.getAttribute("execution.command"));
+ }
+
+ @Test
+ public void testExecuteJarPutToAttribute() throws Exception {
+ File exJar = new File("src/test/resources/ExecuteCommand/TestSuccess.jar");
+ File dummy = new File("src/test/resources/ExecuteCommand/1000bytes.txt");
+ String jarPath = exJar.getAbsolutePath();
+ exJar.setExecutable(true);
+ final TestRunner controller = TestRunners.newTestRunner(ExecuteStreamCommand.class);
+ controller.setValidateExpressionUsage(false);
+ controller.enqueue(dummy.toPath());
+ controller.setProperty(ExecuteStreamCommand.EXECUTION_COMMAND, "java");
+ controller.setProperty(ExecuteStreamCommand.EXECUTION_ARGUMENTS, "-jar;" + jarPath);
+ controller.setProperty(ExecuteStreamCommand.PUT_OUTPUT_IN_ATTRIBUTE, "executeStreamCommand.output");
+ controller.run(1);
+ controller.assertTransferCount(ExecuteStreamCommand.ORIGINAL_RELATIONSHIP, 1);
+ controller.assertTransferCount(ExecuteStreamCommand.OUTPUT_STREAM_RELATIONSHIP, 0);
+
+ List<MockFlowFile> flowFiles = controller.getFlowFilesForRelationship(ExecuteStreamCommand.ORIGINAL_RELATIONSHIP);
+ MockFlowFile outputFlowFile = flowFiles.get(0);
+ String result = outputFlowFile.getAttribute("executeStreamCommand.output");
+ outputFlowFile.assertContentEquals(dummy);
+ assertTrue(Pattern.compile("Test was a success\r?\n").matcher(result).find());
+ assertEquals("0", outputFlowFile.getAttribute("execution.status"));
+ assertEquals("java", outputFlowFile.getAttribute("execution.command"));
+ assertEquals("-jar;", outputFlowFile.getAttribute("execution.command.args").substring(0, 5));
+ String attribute = outputFlowFile.getAttribute("execution.command.args");
+ String expected = "src" + File.separator + "test" + File.separator + "resources" + File.separator + "ExecuteCommand" + File.separator + "TestSuccess.jar";
+ assertEquals(expected, attribute.substring(attribute.length() - expected.length()));
+ }
+
+ @Test
+ public void testExecuteJarToAttributeConfiguration() throws Exception {
+ File exJar = new File("src/test/resources/ExecuteCommand/TestSuccess.jar");
+ String jarPath = exJar.getAbsolutePath();
+ exJar.setExecutable(true);
+ final TestRunner controller = TestRunners.newTestRunner(ExecuteStreamCommand.class);
+ controller.setValidateExpressionUsage(false);
+ controller.enqueue("small test".getBytes());
+ controller.setProperty(ExecuteStreamCommand.EXECUTION_COMMAND, "java");
+ controller.setProperty(ExecuteStreamCommand.EXECUTION_ARGUMENTS, "-jar;" + jarPath);
+ controller.setProperty(ExecuteStreamCommand.PUT_ATTRIBUTE_MAX_LENGTH, "10");
+ controller.setProperty(ExecuteStreamCommand.PUT_OUTPUT_IN_ATTRIBUTE, "outputDest");
+ assertEquals(1, controller.getProcessContext().getAvailableRelationships().size());
+ controller.run(1);
+ controller.assertTransferCount(ExecuteStreamCommand.ORIGINAL_RELATIONSHIP, 1);
+ controller.assertTransferCount(ExecuteStreamCommand.OUTPUT_STREAM_RELATIONSHIP, 0);
+
+ List<MockFlowFile> flowFiles = controller.getFlowFilesForRelationship(ExecuteStreamCommand.ORIGINAL_RELATIONSHIP);
+ MockFlowFile outputFlowFile = flowFiles.get(0);
+ outputFlowFile.assertContentEquals("small test".getBytes());
+ String result = outputFlowFile.getAttribute("outputDest");
+ assertTrue(Pattern.compile("Test was a").matcher(result).find());
+ assertEquals("0", outputFlowFile.getAttribute("execution.status"));
+ assertEquals("java", outputFlowFile.getAttribute("execution.command"));
+ assertEquals("-jar;", outputFlowFile.getAttribute("execution.command.args").substring(0, 5));
+ String attribute = outputFlowFile.getAttribute("execution.command.args");
+ String expected = "src" + File.separator + "test" + File.separator + "resources" + File.separator + "ExecuteCommand" + File.separator + "TestSuccess.jar";
+ assertEquals(expected, attribute.substring(attribute.length() - expected.length()));
+ }
+
+ @Test
+ public void testExecuteIngestAndUpdatePutToAttribute() throws IOException {
+ File exJar = new File("src/test/resources/ExecuteCommand/TestIngestAndUpdate.jar");
+ File dummy = new File("src/test/resources/ExecuteCommand/1000bytes.txt");
+ File dummy100MBytes = new File("target/100MB.txt");
+ FileInputStream fis = new FileInputStream(dummy);
+ FileOutputStream fos = new FileOutputStream(dummy100MBytes);
+ byte[] bytes = new byte[1024];
+ assertEquals(1000, fis.read(bytes));
+ fis.close();
+ for (int i = 0; i < 100000; i++) {
+ fos.write(bytes, 0, 1000);
+ }
+ fos.close();
+ String jarPath = exJar.getAbsolutePath();
+ exJar.setExecutable(true);
+ final TestRunner controller = TestRunners.newTestRunner(ExecuteStreamCommand.class);
+ controller.setValidateExpressionUsage(false);
+ controller.enqueue(dummy100MBytes.toPath());
+ controller.setProperty(ExecuteStreamCommand.EXECUTION_COMMAND, "java");
+ controller.setProperty(ExecuteStreamCommand.EXECUTION_ARGUMENTS, "-jar;" + jarPath);
+ controller.setProperty(ExecuteStreamCommand.PUT_OUTPUT_IN_ATTRIBUTE, "outputDest");
+ controller.run(1);
+ controller.assertTransferCount(ExecuteStreamCommand.ORIGINAL_RELATIONSHIP, 1);
+ controller.assertTransferCount(ExecuteStreamCommand.OUTPUT_STREAM_RELATIONSHIP, 0);
+ List<MockFlowFile> flowFiles = controller.getFlowFilesForRelationship(ExecuteStreamCommand.ORIGINAL_RELATIONSHIP);
+ String result = flowFiles.get(0).getAttribute("outputDest");
+
+ assertTrue(Pattern.compile("nifi-standard-processors:ModifiedResult\r?\n").matcher(result).find());
+ }
+
+ @Test
+ public void testLargePutToAttribute() throws IOException {
+ File dummy = new File("src/test/resources/ExecuteCommand/1000bytes.txt");
+ File dummy100MBytes = new File("target/100MB.txt");
+ FileInputStream fis = new FileInputStream(dummy);
+ FileOutputStream fos = new FileOutputStream(dummy100MBytes);
+ byte[] bytes = new byte[1024];
+ assertEquals(1000, fis.read(bytes));
+ fis.close();
+ for (int i = 0; i < 100000; i++) {
+ fos.write(bytes, 0, 1000);
+ }
+ fos.close();
+
+ final TestRunner controller = TestRunners.newTestRunner(ExecuteStreamCommand.class);
+ controller.setValidateExpressionUsage(false);
+ controller.enqueue("".getBytes());
+ if(isWindows()) {
+ controller.setProperty(ExecuteStreamCommand.EXECUTION_COMMAND, "cmd.exe");
+ controller.setProperty(ExecuteStreamCommand.EXECUTION_ARGUMENTS, "/c;type " + dummy100MBytes.getAbsolutePath());
+ controller.setProperty(ExecuteStreamCommand.ARG_DELIMITER, ";");
+ } else{
+ controller.setProperty(ExecuteStreamCommand.EXECUTION_COMMAND, "cat");
+ controller.setProperty(ExecuteStreamCommand.EXECUTION_ARGUMENTS, dummy100MBytes.getAbsolutePath());
+ }
+ controller.setProperty(ExecuteStreamCommand.IGNORE_STDIN, "true");
+ controller.setProperty(ExecuteStreamCommand.PUT_OUTPUT_IN_ATTRIBUTE, "executeStreamCommand.output");
+ controller.setProperty(ExecuteStreamCommand.PUT_ATTRIBUTE_MAX_LENGTH, "256");
+
+ controller.run(1);
+ controller.assertTransferCount(ExecuteStreamCommand.ORIGINAL_RELATIONSHIP, 1);
+ controller.assertTransferCount(ExecuteStreamCommand.OUTPUT_STREAM_RELATIONSHIP, 0);
+ List<MockFlowFile> flowFiles = controller.getFlowFilesForRelationship(ExecuteStreamCommand.ORIGINAL_RELATIONSHIP);
+
+ flowFiles.get(0).assertAttributeEquals("execution.status", "0");
+ String result = flowFiles.get(0).getAttribute("executeStreamCommand.output");
+ assertTrue(Pattern.compile("a{256}").matcher(result).matches());
+ }
+
+ @Test
+ public void testExecuteIngestAndUpdateWithWorkingDirPutToAttribute() throws IOException {
+ File exJar = new File("src/test/resources/ExecuteCommand/TestIngestAndUpdate.jar");
+ File dummy = new File("src/test/resources/ExecuteCommand/1000bytes.txt");
+ String jarPath = exJar.getAbsolutePath();
+ exJar.setExecutable(true);
+ final TestRunner controller = TestRunners.newTestRunner(ExecuteStreamCommand.class);
+ controller.setValidateExpressionUsage(false);
+
+ controller.enqueue(dummy.toPath());
+ controller.setProperty(ExecuteStreamCommand.WORKING_DIR, "target");
+ controller.setProperty(ExecuteStreamCommand.EXECUTION_COMMAND, "java");
+ controller.setProperty(ExecuteStreamCommand.PUT_OUTPUT_IN_ATTRIBUTE, "streamOutput");
+ controller.setProperty(ExecuteStreamCommand.EXECUTION_ARGUMENTS, "-jar;" + jarPath);
+ controller.run(1);
+ controller.assertTransferCount(ExecuteStreamCommand.ORIGINAL_RELATIONSHIP, 1);
+ List<MockFlowFile> flowFiles = controller.getFlowFilesForRelationship(ExecuteStreamCommand.ORIGINAL_RELATIONSHIP);
+ String result = flowFiles.get(0).getAttribute("streamOutput");
+
+ final String quotedSeparator = Pattern.quote(File.separator);
+ assertTrue(Pattern.compile(quotedSeparator + "nifi-standard-processors" + quotedSeparator + "target:ModifiedResult\r?\n").matcher(result).find());
+ }
+
+ @Test
+ public void testIgnoredStdinPutToAttribute() throws IOException {
+ File exJar = new File("src/test/resources/ExecuteCommand/TestIngestAndUpdate.jar");
+ File dummy = new File("src/test/resources/ExecuteCommand/1000bytes.txt");
+ String jarPath = exJar.getAbsolutePath();
+ exJar.setExecutable(true);
+ final TestRunner controller = TestRunners.newTestRunner(ExecuteStreamCommand.class);
+ controller.setValidateExpressionUsage(false);
+ controller.enqueue(dummy.toPath());
+ controller.setProperty(ExecuteStreamCommand.WORKING_DIR, "target");
+ controller.setProperty(ExecuteStreamCommand.EXECUTION_COMMAND, "java");
+ controller.setProperty(ExecuteStreamCommand.EXECUTION_ARGUMENTS, "-jar;" + jarPath);
+ controller.setProperty(ExecuteStreamCommand.IGNORE_STDIN, "true");
+ controller.setProperty(ExecuteStreamCommand.PUT_OUTPUT_IN_ATTRIBUTE, "executeStreamCommand.output");
+ controller.run(1);
+ controller.assertTransferCount(ExecuteStreamCommand.ORIGINAL_RELATIONSHIP, 1);
+ List<MockFlowFile> flowFiles = controller.getFlowFilesForRelationship(ExecuteStreamCommand.ORIGINAL_RELATIONSHIP);
+ String result = flowFiles.get(0).getAttribute("executeStreamCommand.output");
+ assertTrue("TestIngestAndUpdate.jar should not have received anything to modify",
+ Pattern.compile("target:ModifiedResult\r?\n?").matcher(result).find());
+ }
+
+ @Test
+ public void testDynamicEnvironmentPutToAttribute() throws Exception {
+ File exJar = new File("src/test/resources/ExecuteCommand/TestDynamicEnvironment.jar");
+ File dummy = new File("src/test/resources/ExecuteCommand/1000bytes.txt");
+ String jarPath = exJar.getAbsolutePath();
+ exJar.setExecutable(true);
+ final TestRunner controller = TestRunners.newTestRunner(ExecuteStreamCommand.class);
+ controller.setProperty("NIFI_TEST_1", "testvalue1");
+ controller.setProperty("NIFI_TEST_2", "testvalue2");
+ controller.setValidateExpressionUsage(false);
+ controller.enqueue(dummy.toPath());
+ controller.setProperty(ExecuteStreamCommand.WORKING_DIR, "target");
+ controller.setProperty(ExecuteStreamCommand.EXECUTION_COMMAND, "java");
+ controller.setProperty(ExecuteStreamCommand.EXECUTION_ARGUMENTS, "-jar;" + jarPath);
+ controller.setProperty(ExecuteStreamCommand.PUT_OUTPUT_IN_ATTRIBUTE, "executeStreamCommand.output");
+ controller.run(1);
+ controller.assertTransferCount(ExecuteStreamCommand.ORIGINAL_RELATIONSHIP, 1);
+ List<MockFlowFile> flowFiles = controller.getFlowFilesForRelationship(ExecuteStreamCommand.ORIGINAL_RELATIONSHIP);
+ String result = flowFiles.get(0).getAttribute("executeStreamCommand.output");
+ Set<String> dynamicEnvironmentVariables = new HashSet<>(Arrays.asList(result.split("\r?\n")));
+ assertFalse("Should contain at least two environment variables starting with NIFI", dynamicEnvironmentVariables.size() < 2);
+ assertTrue("NIFI_TEST_1 environment variable is missing", dynamicEnvironmentVariables.contains("NIFI_TEST_1=testvalue1"));
+ assertTrue("NIFI_TEST_2 environment variable is missing", dynamicEnvironmentVariables.contains("NIFI_TEST_2=testvalue2"));
+ }
+
+ @Test
public void testQuotedArguments() throws Exception {
List<String> args = ArgumentUtils.splitArgs("echo -n \"arg1 arg2 arg3\"", ' ');
assertEquals(3, args.size());
@@ -250,4 +478,7 @@ public class TestExecuteStreamCommand {
controller.assertValid();
}
+ private static boolean isWindows() {
+ return System.getProperty("os.name").toLowerCase().startsWith("windows");
+ }
}
[10/50] [abbrv] nifi git commit: Changes after review
Posted by mc...@apache.org.
Changes after review
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/03a54bf2
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/03a54bf2
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/03a54bf2
Branch: refs/heads/NIFI-655
Commit: 03a54bf2d593e07ab602f6a9425d0231a273ba5a
Parents: b954ca6
Author: Naveen Madhire <na...@capitalone.com>
Authored: Mon Nov 16 13:32:17 2015 -0600
Committer: Naveen Madhire <na...@capitalone.com>
Committed: Mon Nov 16 13:32:17 2015 -0600
----------------------------------------------------------------------
.../main/java/org/apache/nifi/processors/kafka/GetKafka.java | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/03a54bf2/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/processors/kafka/GetKafka.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/processors/kafka/GetKafka.java b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/processors/kafka/GetKafka.java
index e644064..4be6194 100644
--- a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/processors/kafka/GetKafka.java
+++ b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/processors/kafka/GetKafka.java
@@ -69,9 +69,7 @@ import kafka.message.MessageAndMetadata;
@WritesAttribute(attribute = "kafka.key", description = "The key of the Kafka message, if it exists and batch size is 1. If"
+ " the message does not have a key, or if the batch size is greater than 1, this attribute will not be added"),
@WritesAttribute(attribute = "kafka.partition", description = "The partition of the Kafka Topic from which the message was received. This attribute is added only if the batch size is 1"),
- @WritesAttribute(attribute = "kafka.offset", description = "The offset of the message within the Kafka partition. This attribute is added only if the batch size is 1"),
- @WritesAttribute(attribute = "auto.offset.reset", description = "If this is set to largest, the consumer may lose some messages when the number of partitions, " +
- "for the topics it subscribes to, changes on the broker. To prevent data loss during partition addition, set auto.offset.reset to smallest")})
+ @WritesAttribute(attribute = "kafka.offset", description = "The offset of the message within the Kafka partition. This attribute is added only if the batch size is 1")})
public class GetKafka extends AbstractProcessor {
public static final String SMALLEST = "smallest";
@@ -155,7 +153,7 @@ public class GetKafka extends AbstractProcessor {
public static final PropertyDescriptor AUTO_OFFSET_RESET = new PropertyDescriptor.Builder()
.name("Auto Offset Reset")
- .description("Auto Offset Reset indicator")
+ .description("Automatically reset the offset to the smallest or largest offset available on the broker")
.required(true)
.allowableValues(SMALLEST, LARGEST)
.defaultValue(LARGEST)
[37/50] [abbrv] nifi git commit: NIFI-1210: Fixes Copy and Paste
descriptions and changes several references to canvas instead of graph to
maintain consistency. Also capitalizes Site-to-Site in a couple of places for
consistency.
Posted by mc...@apache.org.
NIFI-1210: Fixes Copy and Paste descriptions and changes several references to canvas instead of graph to maintain consistency. Also capitalizes Site-to-Site in a couple of places for consistency.
Reviewed by Tony Kurc (tkurc@apache.org)
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/b74617ba
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/b74617ba
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/b74617ba
Branch: refs/heads/NIFI-655
Commit: b74617ba6a18247f0c3ab9ee506ce968d9da51af
Parents: 3273429
Author: Jenn Barnabee <je...@gmail.com>
Authored: Sun Nov 22 08:03:40 2015 -0500
Committer: Tony Kurc <tr...@gmail.com>
Committed: Sun Nov 22 08:29:14 2015 -0500
----------------------------------------------------------------------
nifi-docs/src/main/asciidoc/user-guide.adoc | 52 ++++++++++++------------
1 file changed, 26 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/b74617ba/nifi-docs/src/main/asciidoc/user-guide.adoc
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/user-guide.adoc b/nifi-docs/src/main/asciidoc/user-guide.adoc
index f8efa6f..fd40526 100644
--- a/nifi-docs/src/main/asciidoc/user-guide.adoc
+++ b/nifi-docs/src/main/asciidoc/user-guide.adoc
@@ -37,7 +37,7 @@ use a supported web browser to view the User Interface. Supported web browsers i
* Google Chrome 36+
* Safari 8
-Note that there is a known issue in Internet Explorer (IE) 10 and 11 that can cause problems when moving items on the NiFi graph. If you encounter this problem, we suggest using a browser other than IE. This known issue is described here: https://connect.microsoft.com/IE/Feedback/Details/1050422.
+Note that there is a known issue in Internet Explorer (IE) 10 and 11 that can cause problems when moving items on the NiFi canvas. If you encounter this problem, we suggest using a browser other than IE. This known issue is described here: https://connect.microsoft.com/IE/Feedback/Details/1050422.
[template="glossary", id="terminology"]
@@ -127,8 +127,8 @@ Along the top of the of the screen is a toolbar that contains several of these s
To the left is the Components Toolbar. This toolbar consists of the different components that can be dragged onto the canvas.
Next to the Components Toolbar is the Actions Toolbar. This toolbar consists of buttons to manipulate the existing
-components on the graph. To the right of the Actions Toolbar is the Search Toolbar. This toolbar consists of a single
-Search field that allows users to easily find components on the graph. Users are able to search by component name,
+components on the canvas. To the right of the Actions Toolbar is the Search Toolbar. This toolbar consists of a single
+Search field that allows users to easily find components on the canvas. Users are able to search by component name,
type, identifier, configuration properties, and their values.
The Management Toolbar sits to the right-hand side of the screen. This toolbar consists of buttons that are
@@ -137,8 +137,8 @@ and configure system properties, such as how many system resources should be pro
image::nifi-toolbar-components.png["NiFi Components Toolbar"]
-Next, we have segments that provide capabilities to easily navigate around the graph. On the left-hand side is a toolbar that
-provides the ability to pan around the graph and zoom in and out. On the right-hand side is a “Birds-Eye View” of the dataflow.
+Next, we have segments that provide capabilities to easily navigate around the canvas. On the left-hand side is a toolbar that
+provides the ability to pan around the canvas and zoom in and out. On the right-hand side is a “Birds-Eye View” of the dataflow.
This provides a high-level view of the dataflow and allows the user to quickly and easily pan across large portions of the dataflow.
Along the top of the screen is a trail of breadcrumbs. As users navigate into and out of Process Groups, the breadcrumbs show
the depth in the flow and each Process Group that was entered to reach this depth. Each of the Process Groups listed in the breadcrumbs
@@ -147,8 +147,8 @@ is a link that will take you back up to that level in the flow.
image::nifi-navigation.png["NiFi Navigation"]
[[status_bar]]
-Below the breadcrumbs lives the Status bar. The Status bar provides information about how many Processors exist in the graph in
-each state (Stopped, Running, Invalid, Disabled), how many Remote Process Groups exist on the graph in each state
+Below the breadcrumbs lives the Status bar. The Status bar provides information about how many Processors exist on the canvas in
+each state (Stopped, Running, Invalid, Disabled), how many Remote Process Groups exist on the canvas in each state
(Transmitting, Not Transmitting), the number of threads that are currently active in the flow, the amount of data that currently
exists in the flow, and the timestamp at which all of this information was last refreshed. If there are any System-Level bulletins,
these are shown in the Status bar as well. Additionally, if the instance of NiFi is clustered, the Status bar shows how many nodes
@@ -179,7 +179,7 @@ image:iconProcessor.png["Processor", width=32]
*Processor*: The Processor is the most commonly used component, as it is responsible for data ingress, egress, routing, and
manipulating. There are many different types of Processors. In fact, this is a very common Extension Point in NiFi,
meaning that many vendors may implement their own Processors to perform whatever functions are necessary for their use case.
- When a Processor is dragged onto the graph, the user is presented with a dialog to choose which type of Processor to use:
+ When a Processor is dragged onto the canvas, the user is presented with a dialog to choose which type of Processor to use:
image::add-processor.png["Add Processor Dialog"]
@@ -195,7 +195,7 @@ image::add-processor-with-tag-cloud.png["Add Processor with Tag Cloud"]
Clicking the `Add` button or double-clicking on a Processor Type will add the selected Processor to the canvas at the
location that it was dropped.
-*Note*: For any component added to the canvas, it is possible to select it with the mouse and move it anywhere on the graph. Also, it is possible to select multiple items at once by either holding down the Shift key and selecting each item or by holding down the Shift key and dragging a selection box around the desired components.
+*Note*: For any component added to the canvas, it is possible to select it with the mouse and move it anywhere on the canvas. Also, it is possible to select multiple items at once by either holding down the Shift key and selecting each item or by holding down the Shift key and dragging a selection box around the desired components.
Once a Processor has been dragged onto the canvas, the user may interact with it by right-clicking on the Processor and selecting an option from
context menu.
@@ -212,7 +212,7 @@ The following options are available:
- *Usage*: This option takes the user to the Processor's usage documentation.
- *Change color*: This option allows the user to change the color of the Processor, which can make the visual management of large flows easier.
- *Center in view*: This option centers the view of the canvas on the given Processor.
-- *Copy* or *Paste*: This option places a copy of the selected Processor on the clipboard, so that it may be pasted elsewhere on the canvas.
+- *Copy*: This option places a copy of the selected Processor on the clipboard, so that it may be pasted elsewhere on the canvas by right-clicking on the canvas and selecting Paste. The Copy/Paste actions also may be done using the keystrokes Ctrl-C (Command-C) and Ctrl-V (Command-V).
- *Delete*: This option allows the DFM to delete a Processor from the canvas.
@@ -266,7 +266,7 @@ The following options are available:
- *Upstream connections*: This option allows the user to see and "jump to" upstream connections that are coming into the Process Group.
- *Downstream connections*: This option allows the user to see and "jump to" downstream connections that are going out of the Process Group.
- *Center in view*: This option centers the view of the canvas on the given Process Group.
-- *Copy* or *Paste*: This option places a copy of the selected Process Group on the clipboard, so that it may be pasted elsewhere on the canvas.
+- *Copy*: This option places a copy of the selected Process Group on the clipboard, so that it may be pasted elsewhere on the canvas by right-clicking on the canvas and selecting Paste. The Copy/Paste actions also may be done using the keystrokes Ctrl-C (Command-C) and Ctrl-V (Command-V).
- *Delete*: This option allows the DFM to delete a Process Group.
@@ -290,16 +290,16 @@ image::nifi-rpg-menu.png["Remote Process Group Menu", width=300]
The following options are available:
- *Configure*: This option allows the user to establish or change the configuration of the Remote Process Group.
-- *Remote Ports*: This option allows the user to see input ports and/or output ports that exist on the remote instance of NiFi that the Remote Process Group is connected to. Note that if the site-to-site configuration is secure, only the ports that the connecting NiFi has been given accessed to will be visible.
+- *Remote Ports*: This option allows the user to see input ports and/or output ports that exist on the remote instance of NiFi that the Remote Process Group is connected to. Note that if the Site-to-Site configuration is secure, only the ports that the connecting NiFi has been given accessed to will be visible.
- *Enable transmission*: Makes the transmission of data between NiFi instances active. (See <<Remote_Group_Transmission>> )
- *Disable transmission*: Disables the transmission of data between NiFi instances.
- *Stats*: This option opens a graphical representation of the Remote Process Group's statistical information over time.
- *Upstream connections*: This option allows the user to see and "jump to" upstream connections that are coming into the Remote Process Group.
- *Downstream connections*: This option allows the user to see and "jump to" downstream connections that are going out of the Remote Process Group.
- *Refresh*: This option refreshes the view of the status of the remote NiFi instance.
-- *Go to*: This option opens a view of the remote NiFi instance in a new tab of the browser. Note that if the site-to-site configuration is secure, the user must have access to the remote NiFi instance in order to view it.
+- *Go to*: This option opens a view of the remote NiFi instance in a new tab of the browser. Note that if the Site-to-Site configuration is secure, the user must have access to the remote NiFi instance in order to view it.
- *Center in view*: This option centers the view of the canvas on the given Remote Process Group.
-- *Copy* or *Paste*: This option places a copy of the selected Process Group on the clipboard, so that it may be pasted elsewhere on the canvas.
+- *Copy*: This option places a copy of the selected Process Group on the clipboard, so that it may be pasted elsewhere on the canvas by right-clicking on the canvas and selecting Paste. The Copy/Paste actions also may be done using the keystrokes Ctrl-C (Command-C) and Ctrl-V (Command-V).
- *Delete*: This option allows the DFM to delete a Remote Process Group from the canvas.
@@ -359,7 +359,7 @@ image::settings-tab.png["Settings Tab"]
This tab contains several different configuration items. First, it allows the DFM to change the name of the Processor.
The name of a Processor by default is the same as the Processor type. Next to the Processor Name is a checkbox, indicating
- whether the Processor is Enabled. When a Processor is added to the graph, it is enabled. If the
+ whether the Processor is Enabled. When a Processor is added to the canvas, it is enabled. If the
Processor is disabled, it cannot be started. The disabled state is used to indicate that when a group of Processors is started,
such as when a DFM starts an entire Process Group, this (disabled) Processor should be excluded.
@@ -581,7 +581,7 @@ When you want to run the Reporting Task, click the Start button in the far-right
[[Connecting_Components]]
=== Connecting Components
-Once processors and other components have been added to the graph and configured, the next step is to connect them
+Once processors and other components have been added to the canvas and configured, the next step is to connect them
to one another so that NiFi knows what to do with each FlowFile after it has been processed. This is accomplished by creating a
Connection between each component. When the user hovers the mouse over the center of a component, a new Connection icon (
image:addConnect.png["Connection Bubble"]
@@ -622,7 +622,7 @@ File expiration is a concept by which data that cannot be processed in a timely
This is useful, for example, when the volume of data is expected to exceed the volume that can be sent to a remote site.
In this case, the expiration can be used in conjunction with Prioritizers to ensure that the highest priority data is
processed first and then anything that cannot be processed within a certain time period (one hour, for example) can be dropped. The expiration period is based on the time that the data entered the NiFi instance. In other words, if the file expiration on a given connection is set to '1 hour', and a file that has been in the NiFi instance for one hour reaches that connection, it will expire. The default
-value of `0 sec` indicates that the data will never expire. When a file expiration other than '0 sec' is set, a small clock icon appears on the connection label, so the DFM can see it at-a-glance when looking at a flow on the graph.
+value of `0 sec` indicates that the data will never expire. When a file expiration other than '0 sec' is set, a small clock icon appears on the connection label, so the DFM can see it at-a-glance when looking at a flow on the canvas.
NiFi provides two configuration elements for Back Pressure. These thresholds indicate how much data should be
@@ -730,7 +730,7 @@ link:administration-guide.html[Admin Guide].
This allows new capabilities to be added while still maintaining backward compatibility with all older instances. Additionally, if a vulnerability
or deficiency is ever discovered in a protocol, it allows a newer version of NiFi to forbid communication over the compromised versions of the protocol.
-In order to communicate with a remote NiFi instance via Site-to-Site, simply drag a <<remote_process_group,Remote Process Group>> onto the graph
+In order to communicate with a remote NiFi instance via Site-to-Site, simply drag a <<remote_process_group,Remote Process Group>> onto the canvas
and enter the URL of the remote NiFi instance (for more information on the components of a Remote Process Group, see
<<Remote_Group_Transmission,Remote Process Group Transmission>> section of this guide.) The URL is the same
URL you would use to go to that instance's User Interface. At that point, you can drag a connection to or from the Remote Process Group
@@ -747,7 +747,7 @@ communicate with. For information on configuring NiFi to run securely, see the
link:administration-guide.html[Admin Guide].
In order to allow another NiFi instance to push data to your local instance, you can simply drag an <<input_port,Input Port>> onto the Root Process Group
-of your graph. After entering a name for the port, it will be added to your flow. You can now right-click on the Input Port and choose Configure in order
+of your canvas. After entering a name for the port, it will be added to your flow. You can now right-click on the Input Port and choose Configure in order
to adjust the name and the number of concurrent tasks that are used for the port. If Site-to-Site is configured to run securely, you will also be given
the ability to adjust who has access to the port. If secure, only those who have been granted access to communicate with the port will be able to see
that the port exists.
@@ -781,7 +781,7 @@ This section has described the steps required to build a dataflow. Now, to put i
consists of just two processors: GenerateFlowFile and LogAttribute. These processors are normally used for testing, but they can also be used
to build a quick flow for demonstration purposes and see NiFi in action.
-After you drag the GenerateFlowFile and LogAttribute processors to the graph and connect them (using the guidelines provided above), configure them as follows:
+After you drag the GenerateFlowFile and LogAttribute processors to the canvas and connect them (using the guidelines provided above), configure them as follows:
* Generate FlowFile
** On the Scheduling tab, set Run schedule to: 5 sec. Note that the GenerateFlowFile processor can create many FlowFiles very quickly; that's why setting the Run schedule is important so that this flow does not overwhelm the system NiFi is running on.
@@ -940,7 +940,7 @@ or not compression should be used when transmitting data to or from this Port.
== Navigating within a DataFlow
NiFi provides various mechanisms for getting around a dataflow. The <<User_Interface>> section discussed various ways to navigate around
-the NiFi graph; however, once a flow exists on the graph, there are additional ways to get from one component to another. The <<User Interface>> section showed that when multiple Process Groups exist in a flow, breadcrumbs appear under the toolbar, providing a way to navigate between them. In addition, to enter a Process Group that is currently visible on the graph, simply double-click it, thereby "drilling down" into it. Connections also provide a way to jump from one location to another within the flow. Right-click on a connection and select "Go to source" or "Go to destination" in order to jump to one end of the connection or another. This can be very useful in large, complex dataflows, where the connection lines may be long and span large areas of the graph. Finally, all components provide the ability to jump forward or backward within the flow. Right-click any component (e.g., a processor, process group, port, etc.) and select either "Upstream connections" or "Downstream connec
tions". A dialog window will open, showing the available upstream or downstream connections that the user may jump to. This can be especially useful when trying to follow a dataflow in a backward direction. It is typically easy to follow the path of a dataflow from start to finish, drilling down into nested process groups; however, it can be more difficult to follow the dataflow in the other direction.
+the NiFi canvas; however, once a flow exists on the canvas, there are additional ways to get from one component to another. The <<User Interface>> section showed that when multiple Process Groups exist in a flow, breadcrumbs appear under the toolbar, providing a way to navigate between them. In addition, to enter a Process Group that is currently visible on the canvas, simply double-click it, thereby "drilling down" into it. Connections also provide a way to jump from one location to another within the flow. Right-click on a connection and select "Go to source" or "Go to destination" in order to jump to one end of the connection or another. This can be very useful in large, complex dataflows, where the connection lines may be long and span large areas of the canvas. Finally, all components provide the ability to jump forward or backward within the flow. Right-click any component (e.g., a processor, process group, port, etc.) and select either "Upstream connections" or "Downstream co
nnections". A dialog window will open, showing the available upstream or downstream connections that the user may jump to. This can be especially useful when trying to follow a dataflow in a backward direction. It is typically easy to follow the path of a dataflow from start to finish, drilling down into nested process groups; however, it can be more difficult to follow the dataflow in the other direction.
@@ -952,7 +952,7 @@ health and status. The Status bar provides information about the overall system
(See <<status_bar>> above for more information). Processors, Process Groups, and Remote Process Groups
provide fine-grained details about their operations. Connections and Process Groups provide information
about the amount of data in their queues. The Summary Page provides information about all of the components
-on the graph in a tabular format and also provides System Diagnostics information that includes disk usage,
+on the canvas in a tabular format and also provides System Diagnostics information that includes disk usage,
CPU utilization, and Java Heap and Garbage Collection information. In a clustered environment, this
information is available per-node or as aggregates across the entire cluster. We will explore each of these
monitoring artifacts below.
@@ -1241,7 +1241,7 @@ image:iconSummary.png["Summary"]
image::summary-table.png["Summary Table"]
-This dialog provides a great deal of information about each of the components on the graph. Below, we have annotated
+This dialog provides a great deal of information about each of the components on the canvas. Below, we have annotated
the different elements within the dialog in order to make the discussion of the dialog easier.
image::summary-annotated.png["Summary Table Annotated"]
@@ -1377,7 +1377,7 @@ error message if unable to create the template for some reason.
.Note
********************************************************************************************************************
It is important to note that if any Processor that is Templated has a sensitive property (such as a password), the value of that
-sensitive property is not included in the Template. As a result, when dragging the Template onto the graph, newly
+sensitive property is not included in the Template. As a result, when dragging the Template onto the canvas, newly
created Processors may not be valid if they are missing values for their sensitive properties. Additionally, any
Connection that was selected when making the Template is not included in the Template if either the source or the
destination of the Connection is not also included in the Template.
@@ -1541,9 +1541,9 @@ image:expanded-events.png["Expanded Events", width=300]
Other Management Features
-------------------------
-In addition to the Summary Page, Data Provenance Page, Template Management Page, and Bulletin Board Page, there are other tools in the Management Toolbar (See <<User_Interface>>) that are useful to the DFM. The Flow Configuration History, which is available by clicking on the clock icon ( image:iconFlowHistory.png["Flow History", width=28] ) in the Management Toolbar, shows all the changes that have been made to the dataflow graph. The history can aid in troubleshooting, such as if a recent change to the dataflow has caused a problem and needs to be fixed. The DFM can see what changes have been made and adjust the flow as needed to fix the problem. While NiFi does not have an "undo" feature, the DFM can make new changes to the dataflow that will fix the problem.
+In addition to the Summary Page, Data Provenance Page, Template Management Page, and Bulletin Board Page, there are other tools in the Management Toolbar (See <<User_Interface>>) that are useful to the DFM. The Flow Configuration History, which is available by clicking on the clock icon ( image:iconFlowHistory.png["Flow History", width=28] ) in the Management Toolbar, shows all the changes that have been made to the dataflow. The history can aid in troubleshooting, such as if a recent change to the dataflow has caused a problem and needs to be fixed. The DFM can see what changes have been made and adjust the flow as needed to fix the problem. While NiFi does not have an "undo" feature, the DFM can make new changes to the dataflow that will fix the problem.
-Two other tools in the Management Toolbar are the Controller Settings page ( image:iconSettings.png["Settings", width=28] ) and the Users page ( image:iconUsers.png["Users", width=28] ). The Controller Settings page provides the ability to change the name of the NiFi instance, add comments describing the NiFi instance, set the maximum number of threads that are available to the application, and create a back-up copy of the dataflow(s) currently on the graph. It also provides tabs where DFMs may add and configure Controller Services and Reporting Tasks (see <<Controller_Services_and_Reporting_Tasks>>). The Users page is used to manage user access, which is described in the Admin Guide.
+Two other tools in the Management Toolbar are the Controller Settings page ( image:iconSettings.png["Settings", width=28] ) and the Users page ( image:iconUsers.png["Users", width=28] ). The Controller Settings page provides the ability to change the name of the NiFi instance, add comments describing the NiFi instance, set the maximum number of threads that are available to the application, and create a back-up copy of the dataflow(s) currently on the canvas. It also provides tabs where DFMs may add and configure Controller Services and Reporting Tasks (see <<Controller_Services_and_Reporting_Tasks>>). The Users page is used to manage user access, which is described in the Admin Guide.
[34/50] [abbrv] nifi git commit: NIFI-1086: Changed behavior on
retrieval with no input file to RETRIEVE events, removed @TriggerWhenEmpty
Posted by mc...@apache.org.
NIFI-1086: Changed behavior on retrieval with no input file to RETRIEVE events, removed @TriggerWhenEmpty
Reviewed by Tony Kurc (tkurc@apache.org)
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/cdd2c4f2
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/cdd2c4f2
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/cdd2c4f2
Branch: refs/heads/NIFI-655
Commit: cdd2c4f22cf9d309aff6ea2fa57f9557bc243006
Parents: f1f67f6
Author: Joseph Percivall <jo...@yahoo.com>
Authored: Sat Nov 21 16:04:13 2015 -0500
Committer: Tony Kurc <tr...@gmail.com>
Committed: Sat Nov 21 19:20:33 2015 -0500
----------------------------------------------------------------------
.../nifi/processors/standard/InvokeHTTP.java | 19 +++++++++----------
1 file changed, 9 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/cdd2c4f2/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/InvokeHTTP.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/InvokeHTTP.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/InvokeHTTP.java
index 2a9760d..6d38bce 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/InvokeHTTP.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/InvokeHTTP.java
@@ -67,7 +67,6 @@ import org.apache.nifi.annotation.behavior.DynamicProperty;
import org.apache.nifi.annotation.behavior.InputRequirement;
import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
import org.apache.nifi.annotation.behavior.SupportsBatching;
-import org.apache.nifi.annotation.behavior.TriggerWhenEmpty;
import org.apache.nifi.annotation.behavior.WritesAttribute;
import org.apache.nifi.annotation.behavior.WritesAttributes;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
@@ -96,7 +95,6 @@ import org.joda.time.format.DateTimeFormatter;
@SupportsBatching
@Tags({"http", "https", "rest", "client"})
@InputRequirement(Requirement.INPUT_ALLOWED)
-@TriggerWhenEmpty
@CapabilityDescription("An HTTP client processor which converts FlowFile attributes to HTTP headers, with configurable HTTP method, url, etc.")
@WritesAttributes({
@WritesAttribute(attribute = "invokehttp.status.code", description = "The status code that is returned"),
@@ -610,7 +608,11 @@ public final class InvokeHTTP extends AbstractProcessor {
// emit provenance event
final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
- session.getProvenanceReporter().fetch(responseFlowFile, url.toExternalForm(), millis);
+ if(requestFlowFile != null) {
+ session.getProvenanceReporter().fetch(responseFlowFile, url.toExternalForm(), millis);
+ } else {
+ session.getProvenanceReporter().receive(responseFlowFile, url.toExternalForm(), millis);
+ }
}
}
@@ -775,13 +777,9 @@ public final class InvokeHTTP extends AbstractProcessor {
private void route(FlowFile request, FlowFile response, ProcessSession session, ProcessContext context, int statusCode){
- // check if we should penalize the request
- if (!isSuccess(statusCode)) {
- if (request == null) {
- context.yield();
- } else {
- request = session.penalize(request);
- }
+ // check if we should yield the processor
+ if (!isSuccess(statusCode) && request == null) {
+ context.yield();
}
// If the property to output the response flowfile regardless of status code is set then transfer it
@@ -805,6 +803,7 @@ public final class InvokeHTTP extends AbstractProcessor {
// 5xx -> RETRY
} else if (statusCode / 100 == 5) {
if (request != null) {
+ request = session.penalize(request);
session.transfer(request, REL_RETRY);
}
[12/50] [abbrv] nifi git commit: NIFI-1114: Added documentation for
nifi.kerberos.krb5.file property
Posted by mc...@apache.org.
NIFI-1114: Added documentation for nifi.kerberos.krb5.file property
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/d16392c6
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/d16392c6
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/d16392c6
Branch: refs/heads/NIFI-655
Commit: d16392c6182678119b77de44c910a481305ce990
Parents: f98e335
Author: Mark Payne <ma...@hotmail.com>
Authored: Tue Nov 17 08:45:28 2015 -0500
Committer: Mark Payne <ma...@hotmail.com>
Committed: Tue Nov 17 08:45:28 2015 -0500
----------------------------------------------------------------------
nifi-docs/src/main/asciidoc/administration-guide.adoc | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/d16392c6/nifi-docs/src/main/asciidoc/administration-guide.adoc
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/administration-guide.adoc b/nifi-docs/src/main/asciidoc/administration-guide.adoc
index 57cee4e..b36c28e 100644
--- a/nifi-docs/src/main/asciidoc/administration-guide.adoc
+++ b/nifi-docs/src/main/asciidoc/administration-guide.adoc
@@ -769,7 +769,9 @@ Only configure these properties for the cluster manager.
|====
|*Property*|*Description*
-|nifi.kerberos.krb5.file*|The location of the krb5 file, if used. It is blank by default.
+|nifi.kerberos.krb5.file*|The location of the krb5 file, if used. It is blank by default. Note that this property is not used to authenticate NiFi users.
+ Rather, it is made available for extension points, such as Hadoop-based Processors, to use. At this time, only a single krb5 file is allowed to
+ be specified per NiFi instance, so this property is configured here rather than in individual Processors.
|====
[31/50] [abbrv] nifi git commit: NIFI-1174 Refactoring the HBase
client API and adding a PutHBaseJSON which can write a whole row from a
single json document - Adding Complex Field Strategy to PutHBaseJSON to allow
more control of complex fields - Improv
Posted by mc...@apache.org.
NIFI-1174 Refactoring the HBase client API and adding a PutHBaseJSON which can write a whole row from a single json document - Adding Complex Field Strategy to PutHBaseJSON to allow more control of complex fields - Improving error messages to indicate what the problem was with an invalid row
Signed-off-by: Bryan Bende <bb...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/40dd8a0a
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/40dd8a0a
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/40dd8a0a
Branch: refs/heads/NIFI-655
Commit: 40dd8a0a845ef5f4d4fde451f02376ab2fab9758
Parents: 8c2323d
Author: Bryan Bende <bb...@apache.org>
Authored: Wed Nov 18 17:24:49 2015 -0500
Committer: Bryan Bende <bb...@apache.org>
Committed: Thu Nov 19 13:49:02 2015 -0500
----------------------------------------------------------------------
.../nifi-hbase-processors/pom.xml | 4 +
.../nifi/hbase/AbstractHBaseProcessor.java | 23 -
.../org/apache/nifi/hbase/AbstractPutHBase.java | 183 ++++++++
.../java/org/apache/nifi/hbase/GetHBase.java | 3 +-
.../org/apache/nifi/hbase/PutHBaseCell.java | 153 +------
.../org/apache/nifi/hbase/PutHBaseJSON.java | 230 ++++++++++
.../org.apache.nifi.processor.Processor | 3 +-
.../org/apache/nifi/hbase/HBaseTestUtil.java | 87 ++++
.../nifi/hbase/MockHBaseClientService.java | 14 +-
.../org/apache/nifi/hbase/TestPutHBaseCell.java | 60 ++-
.../org/apache/nifi/hbase/TestPutHBaseJSON.java | 423 +++++++++++++++++++
.../apache/nifi/hbase/HBaseClientService.java | 11 +
.../org/apache/nifi/hbase/put/PutColumn.java | 47 +++
.../org/apache/nifi/hbase/put/PutFlowFile.java | 38 +-
.../nifi/hbase/HBase_1_1_2_ClientService.java | 25 +-
.../hbase/TestHBase_1_1_2_ClientService.java | 27 +-
16 files changed, 1119 insertions(+), 212 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/40dd8a0a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/pom.xml b/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/pom.xml
index b474c6a..abbe4c9 100644
--- a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/pom.xml
+++ b/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/pom.xml
@@ -50,6 +50,10 @@
<artifactId>commons-lang3</artifactId>
<version>3.4</version>
</dependency>
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-mapper-asl</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
http://git-wip-us.apache.org/repos/asf/nifi/blob/40dd8a0a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/java/org/apache/nifi/hbase/AbstractHBaseProcessor.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/java/org/apache/nifi/hbase/AbstractHBaseProcessor.java b/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/java/org/apache/nifi/hbase/AbstractHBaseProcessor.java
deleted file mode 100644
index 9cce35e..0000000
--- a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/java/org/apache/nifi/hbase/AbstractHBaseProcessor.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.hbase;
-
-import org.apache.nifi.processor.AbstractProcessor;
-
-public abstract class AbstractHBaseProcessor extends AbstractProcessor {
-
-}
http://git-wip-us.apache.org/repos/asf/nifi/blob/40dd8a0a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/java/org/apache/nifi/hbase/AbstractPutHBase.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/java/org/apache/nifi/hbase/AbstractPutHBase.java b/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/java/org/apache/nifi/hbase/AbstractPutHBase.java
new file mode 100644
index 0000000..87424f9
--- /dev/null
+++ b/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/java/org/apache/nifi/hbase/AbstractPutHBase.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hbase;
+
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.hbase.put.PutFlowFile;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Base class for processors that put data to HBase.
+ */
+public abstract class AbstractPutHBase extends AbstractProcessor {
+
+ protected static final PropertyDescriptor HBASE_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("HBase Client Service")
+ .description("Specifies the Controller Service to use for accessing HBase.")
+ .required(true)
+ .identifiesControllerService(HBaseClientService.class)
+ .build();
+ protected static final PropertyDescriptor TABLE_NAME = new PropertyDescriptor.Builder()
+ .name("Table Name")
+ .description("The name of the HBase Table to put data into")
+ .required(true)
+ .expressionLanguageSupported(true)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+ protected static final PropertyDescriptor ROW_ID = new PropertyDescriptor.Builder()
+ .name("Row Identifier")
+ .description("Specifies the Row ID to use when inserting data into HBase")
+ .required(false) // not all sub-classes will require this
+ .expressionLanguageSupported(true)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+ protected static final PropertyDescriptor COLUMN_FAMILY = new PropertyDescriptor.Builder()
+ .name("Column Family")
+ .description("The Column Family to use when inserting data into HBase")
+ .required(true)
+ .expressionLanguageSupported(true)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+ protected static final PropertyDescriptor COLUMN_QUALIFIER = new PropertyDescriptor.Builder()
+ .name("Column Qualifier")
+ .description("The Column Qualifier to use when inserting data into HBase")
+ .required(true)
+ .expressionLanguageSupported(true)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+ protected static final PropertyDescriptor BATCH_SIZE = new PropertyDescriptor.Builder()
+ .name("Batch Size")
+ .description("The maximum number of FlowFiles to process in a single execution. The FlowFiles will be " +
+ "grouped by table, and a single Put per table will be performed.")
+ .required(true)
+ .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR)
+ .defaultValue("25")
+ .build();
+
+ protected static final Relationship REL_SUCCESS = new Relationship.Builder()
+ .name("success")
+ .description("A FlowFile is routed to this relationship after it has been successfully stored in HBase")
+ .build();
+ protected static final Relationship REL_FAILURE = new Relationship.Builder()
+ .name("failure")
+ .description("A FlowFile is routed to this relationship if it cannot be sent to HBase")
+ .build();
+
+ @Override
+ public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
+ final int batchSize = context.getProperty(BATCH_SIZE).asInteger();
+ List<FlowFile> flowFiles = session.get(batchSize);
+ if (flowFiles == null || flowFiles.size() == 0) {
+ return;
+ }
+
+ final Map<String,List<PutFlowFile>> tablePuts = new HashMap<>();
+
+ // Group FlowFiles by HBase Table
+ for (final FlowFile flowFile : flowFiles) {
+ final PutFlowFile putFlowFile = createPut(session, context, flowFile);
+
+ if (putFlowFile == null) {
+ // sub-classes should log appropriate error messages before returning null
+ session.transfer(flowFile, REL_FAILURE);
+ } else if (!putFlowFile.isValid()) {
+ if (StringUtils.isBlank(putFlowFile.getTableName())) {
+ getLogger().error("Missing table name for FlowFile {}; routing to failure", new Object[]{flowFile});
+ } else if (StringUtils.isBlank(putFlowFile.getRow())) {
+ getLogger().error("Missing row id for FlowFile {}; routing to failure", new Object[]{flowFile});
+ } else if (putFlowFile.getColumns() == null || putFlowFile.getColumns().isEmpty()) {
+ getLogger().error("No columns provided for FlowFile {}; routing to failure", new Object[]{flowFile});
+ } else {
+ // really shouldn't get here, but just in case
+ getLogger().error("Failed to produce a put for FlowFile {}; routing to failure", new Object[]{flowFile});
+ }
+ session.transfer(flowFile, REL_FAILURE);
+ } else {
+ List<PutFlowFile> putFlowFiles = tablePuts.get(putFlowFile.getTableName());
+ if (putFlowFiles == null) {
+ putFlowFiles = new ArrayList<>();
+ tablePuts.put(putFlowFile.getTableName(), putFlowFiles);
+ }
+ putFlowFiles.add(putFlowFile);
+ }
+ }
+
+ getLogger().debug("Sending {} FlowFiles to HBase in {} put operations", new Object[]{flowFiles.size(), tablePuts.size()});
+
+ final long start = System.nanoTime();
+ final List<PutFlowFile> successes = new ArrayList<>();
+ final HBaseClientService hBaseClientService = context.getProperty(HBASE_CLIENT_SERVICE).asControllerService(HBaseClientService.class);
+
+ for (Map.Entry<String, List<PutFlowFile>> entry : tablePuts.entrySet()) {
+ try {
+ hBaseClientService.put(entry.getKey(), entry.getValue());
+ successes.addAll(entry.getValue());
+ } catch (Exception e) {
+ getLogger().error(e.getMessage(), e);
+
+ for (PutFlowFile putFlowFile : entry.getValue()) {
+ getLogger().error("Failed to send {} to HBase due to {}; routing to failure", new Object[]{putFlowFile.getFlowFile(), e});
+ final FlowFile failure = session.penalize(putFlowFile.getFlowFile());
+ session.transfer(failure, REL_FAILURE);
+ }
+ }
+ }
+
+ final long sendMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
+ getLogger().debug("Sent {} FlowFiles to HBase successfully in {} milliseconds", new Object[]{successes.size(), sendMillis});
+
+ for (PutFlowFile putFlowFile : successes) {
+ session.transfer(putFlowFile.getFlowFile(), REL_SUCCESS);
+ final String details = "Put " + putFlowFile.getColumns().size() + " cells to HBase";
+ session.getProvenanceReporter().send(putFlowFile.getFlowFile(), getTransitUri(putFlowFile), details, sendMillis);
+ }
+
+ }
+
+ protected String getTransitUri(PutFlowFile putFlowFile) {
+ return "hbase://" + putFlowFile.getTableName() + "/" + putFlowFile.getRow();
+ }
+
+ /**
+ * Sub-classes provide the implementation to create a put from a FlowFile.
+ *
+ * @param session
+ * the current session
+ * @param context
+ * the current context
+ * @param flowFile
+ * the FlowFile to create a Put from
+ *
+ * @return a PutFlowFile instance for the given FlowFile
+ */
+ protected abstract PutFlowFile createPut(final ProcessSession session, final ProcessContext context, final FlowFile flowFile);
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/40dd8a0a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/java/org/apache/nifi/hbase/GetHBase.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/java/org/apache/nifi/hbase/GetHBase.java b/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/java/org/apache/nifi/hbase/GetHBase.java
index 5f08265..98a612c 100644
--- a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/java/org/apache/nifi/hbase/GetHBase.java
+++ b/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/java/org/apache/nifi/hbase/GetHBase.java
@@ -41,6 +41,7 @@ import org.apache.nifi.hbase.scan.ResultCell;
import org.apache.nifi.hbase.scan.ResultHandler;
import org.apache.nifi.hbase.util.ObjectSerDe;
import org.apache.nifi.hbase.util.StringSerDe;
+import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.Relationship;
@@ -83,7 +84,7 @@ import java.util.regex.Pattern;
@WritesAttribute(attribute = "hbase.table", description = "The name of the HBase table that the data was pulled from"),
@WritesAttribute(attribute = "mime.type", description = "Set to application/json to indicate that output is JSON")
})
-public class GetHBase extends AbstractHBaseProcessor {
+public class GetHBase extends AbstractProcessor {
static final Pattern COLUMNS_PATTERN = Pattern.compile("\\w+(:\\w+)?(?:,\\w+(:\\w+)?)*");
http://git-wip-us.apache.org/repos/asf/nifi/blob/40dd8a0a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/java/org/apache/nifi/hbase/PutHBaseCell.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/java/org/apache/nifi/hbase/PutHBaseCell.java b/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/java/org/apache/nifi/hbase/PutHBaseCell.java
index 0a2b763..759d91e 100644
--- a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/java/org/apache/nifi/hbase/PutHBaseCell.java
+++ b/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/java/org/apache/nifi/hbase/PutHBaseCell.java
@@ -16,7 +16,6 @@
*/
package org.apache.nifi.hbase;
-import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.annotation.behavior.EventDriven;
import org.apache.nifi.annotation.behavior.InputRequirement;
import org.apache.nifi.annotation.behavior.SupportsBatching;
@@ -24,91 +23,36 @@ import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.hbase.put.PutColumn;
import org.apache.nifi.hbase.put.PutFlowFile;
-import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.Relationship;
-import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.io.InputStreamCallback;
-import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.stream.io.StreamUtils;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
-import java.util.HashMap;
+import java.util.Collection;
+import java.util.Collections;
import java.util.HashSet;
import java.util.List;
-import java.util.Map;
import java.util.Set;
-import java.util.concurrent.TimeUnit;
@EventDriven
@SupportsBatching
@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED)
@Tags({"hadoop", "hbase"})
@CapabilityDescription("Adds the Contents of a FlowFile to HBase as the value of a single cell")
-public class PutHBaseCell extends AbstractProcessor {
-
- protected static final PropertyDescriptor HBASE_CLIENT_SERVICE = new PropertyDescriptor.Builder()
- .name("HBase Client Service")
- .description("Specifies the Controller Service to use for accessing HBase.")
- .required(true)
- .identifiesControllerService(HBaseClientService.class)
- .build();
- protected static final PropertyDescriptor TABLE_NAME = new PropertyDescriptor.Builder()
- .name("Table Name")
- .description("The name of the HBase Table to put data into")
- .required(true)
- .expressionLanguageSupported(true)
- .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
- .build();
- static final PropertyDescriptor ROW = new PropertyDescriptor.Builder()
- .name("Row Identifier")
- .description("Specifies the Row ID to use when inserting data into HBase")
- .required(true)
- .expressionLanguageSupported(true)
- .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
- .build();
- static final PropertyDescriptor COLUMN_FAMILY = new PropertyDescriptor.Builder()
- .name("Column Family")
- .description("The Column Family to use when inserting data into HBase")
- .required(true)
- .expressionLanguageSupported(true)
- .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
- .build();
- static final PropertyDescriptor COLUMN_QUALIFIER = new PropertyDescriptor.Builder()
- .name("Column Qualifier")
- .description("The Column Qualifier to use when inserting data into HBase")
- .required(true)
- .expressionLanguageSupported(true)
- .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
- .build();
- static final PropertyDescriptor BATCH_SIZE = new PropertyDescriptor.Builder()
- .name("Batch Size")
- .description("The maximum number of FlowFiles to process in a single execution. The FlowFiles will be " +
- "grouped by table, and a single Put per table will be performed.")
- .required(true)
- .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR)
- .defaultValue("25")
- .build();
-
- static final Relationship REL_SUCCESS = new Relationship.Builder()
- .name("success")
- .description("A FlowFile is routed to this relationship after it has been successfully stored in HBase")
- .build();
- static final Relationship FAILURE = new Relationship.Builder()
- .name("failure")
- .description("A FlowFile is routed to this relationship if it cannot be sent to HBase")
- .build();
+public class PutHBaseCell extends AbstractPutHBase {
@Override
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
final List<PropertyDescriptor> properties = new ArrayList<>();
properties.add(HBASE_CLIENT_SERVICE);
properties.add(TABLE_NAME);
- properties.add(ROW);
+ properties.add(ROW_ID);
properties.add(COLUMN_FAMILY);
properties.add(COLUMN_QUALIFIER);
properties.add(BATCH_SIZE);
@@ -119,84 +63,27 @@ public class PutHBaseCell extends AbstractProcessor {
public Set<Relationship> getRelationships() {
final Set<Relationship> rels = new HashSet<>();
rels.add(REL_SUCCESS);
- rels.add(FAILURE);
+ rels.add(REL_FAILURE);
return rels;
}
@Override
- public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
- final int batchSize = context.getProperty(BATCH_SIZE).asInteger();
- List<FlowFile> flowFiles = session.get(batchSize);
- if (flowFiles == null || flowFiles.size() == 0) {
- return;
- }
-
- final Map<String,List<PutFlowFile>> tablePuts = new HashMap<>();
-
- // Group FlowFiles by HBase Table
- for (final FlowFile flowFile : flowFiles) {
- final String tableName = context.getProperty(TABLE_NAME).evaluateAttributeExpressions(flowFile).getValue();
- final String row = context.getProperty(ROW).evaluateAttributeExpressions(flowFile).getValue();
- final String columnFamily = context.getProperty(COLUMN_FAMILY).evaluateAttributeExpressions(flowFile).getValue();
- final String columnQualifier = context.getProperty(COLUMN_QUALIFIER).evaluateAttributeExpressions(flowFile).getValue();
-
- if (StringUtils.isBlank(tableName) || StringUtils.isBlank(row) || StringUtils.isBlank(columnFamily) || StringUtils.isBlank(columnQualifier)) {
- getLogger().error("Invalid FlowFile {} missing table, row, column familiy, or column qualifier; routing to failure", new Object[]{flowFile});
- session.transfer(flowFile, FAILURE);
- } else {
- final byte[] buffer = new byte[(int) flowFile.getSize()];
- session.read(flowFile, new InputStreamCallback() {
- @Override
- public void process(final InputStream in) throws IOException {
- StreamUtils.fillBuffer(in, buffer);
- }
- });
-
- final PutFlowFile putFlowFile = new PutFlowFile(tableName, row, columnFamily, columnQualifier, buffer, flowFile);
-
- List<PutFlowFile> putFlowFiles = tablePuts.get(tableName);
- if (putFlowFiles == null) {
- putFlowFiles = new ArrayList<>();
- tablePuts.put(tableName, putFlowFiles);
- }
- putFlowFiles.add(putFlowFile);
- }
- }
-
- getLogger().debug("Sending {} FlowFiles to HBase in {} put operations", new Object[] {flowFiles.size(), tablePuts.size()});
-
- final long start = System.nanoTime();
- final List<PutFlowFile> successes = new ArrayList<>();
- final HBaseClientService hBaseClientService = context.getProperty(HBASE_CLIENT_SERVICE).asControllerService(HBaseClientService.class);
-
- for (Map.Entry<String, List<PutFlowFile>> entry : tablePuts.entrySet()) {
- try {
- hBaseClientService.put(entry.getKey(), entry.getValue());
- successes.addAll(entry.getValue());
- } catch (Exception e) {
- getLogger().error(e.getMessage(), e);
-
- for (PutFlowFile putFlowFile : entry.getValue()) {
- getLogger().error("Failed to send {} to HBase due to {}; routing to failure", new Object[]{putFlowFile.getFlowFile(), e});
- final FlowFile failure = session.penalize(putFlowFile.getFlowFile());
- session.transfer(failure, FAILURE);
- }
+ protected PutFlowFile createPut(final ProcessSession session, final ProcessContext context, final FlowFile flowFile) {
+ final String tableName = context.getProperty(TABLE_NAME).evaluateAttributeExpressions(flowFile).getValue();
+ final String row = context.getProperty(ROW_ID).evaluateAttributeExpressions(flowFile).getValue();
+ final String columnFamily = context.getProperty(COLUMN_FAMILY).evaluateAttributeExpressions(flowFile).getValue();
+ final String columnQualifier = context.getProperty(COLUMN_QUALIFIER).evaluateAttributeExpressions(flowFile).getValue();
+
+ final byte[] buffer = new byte[(int) flowFile.getSize()];
+ session.read(flowFile, new InputStreamCallback() {
+ @Override
+ public void process(final InputStream in) throws IOException {
+ StreamUtils.fillBuffer(in, buffer);
}
- }
-
- final long sendMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
- getLogger().debug("Sent {} FlowFiles to HBase successfully in {} milliseconds", new Object[] {successes.size(), sendMillis});
-
- for (PutFlowFile putFlowFile : successes) {
- session.transfer(putFlowFile.getFlowFile(), REL_SUCCESS);
- session.getProvenanceReporter().send(putFlowFile.getFlowFile(), getTransitUri(putFlowFile), sendMillis);
- }
-
- }
+ });
- protected String getTransitUri(PutFlowFile putFlowFile) {
- return "hbase://" + putFlowFile.getTableName() + "/" + putFlowFile.getRow() + "/" + putFlowFile.getColumnFamily()
- + ":" + putFlowFile.getColumnQualifier();
+ final Collection<PutColumn> columns = Collections.singletonList(new PutColumn(columnFamily, columnQualifier, buffer));
+ return new PutFlowFile(tableName, row, columns, flowFile);
}
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/40dd8a0a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/java/org/apache/nifi/hbase/PutHBaseJSON.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/java/org/apache/nifi/hbase/PutHBaseJSON.java b/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/java/org/apache/nifi/hbase/PutHBaseJSON.java
new file mode 100644
index 0000000..0dba7ee
--- /dev/null
+++ b/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/java/org/apache/nifi/hbase/PutHBaseJSON.java
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hbase;
+
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.components.AllowableValue;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.hbase.put.PutColumn;
+import org.apache.nifi.hbase.put.PutFlowFile;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.io.InputStreamCallback;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.util.ObjectHolder;
+import org.codehaus.jackson.JsonNode;
+import org.codehaus.jackson.map.ObjectMapper;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+@EventDriven
+@SupportsBatching
+@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED)
+@Tags({"hadoop", "hbase", "put", "json"})
+@CapabilityDescription("Adds rows to HBase based on the contents of incoming JSON documents. Each FlowFile must contain a single " +
+ "UTF-8 encoded JSON document, and any FlowFiles where the root element is not a single document will be routed to failure. " +
+ "Each JSON field name and value will become a column qualifier and value of the HBase row. Any fields with a null value " +
+ "will be skipped, and fields with a complex value will be handled according to the Complex Field Strategy. " +
+ "The row id can be specified either directly on the processor through the Row Identifier property, or can be extracted from the JSON " +
+ "document by specifying the Row Identifier Field Name property. This processor will hold the contents of all FlowFiles for the given batch " +
+ "in memory at one time.")
+public class PutHBaseJSON extends AbstractPutHBase {
+
+ protected static final PropertyDescriptor ROW_FIELD_NAME = new PropertyDescriptor.Builder()
+ .name("Row Identifier Field Name")
+ .description("Specifies the name of a JSON element whose value should be used as the row id for the given JSON document.")
+ .expressionLanguageSupported(true)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+
+ protected static final String FAIL_VALUE = "Fail";
+ protected static final String WARN_VALUE = "Warn";
+ protected static final String IGNORE_VALUE = "Ignore";
+ protected static final String TEXT_VALUE = "Text";
+
+ protected static final AllowableValue COMPLEX_FIELD_FAIL = new AllowableValue(FAIL_VALUE, FAIL_VALUE, "Route entire FlowFile to failure if any elements contain complex values.");
+ protected static final AllowableValue COMPLEX_FIELD_WARN = new AllowableValue(WARN_VALUE, WARN_VALUE, "Provide a warning and do not include field in row sent to HBase.");
+ protected static final AllowableValue COMPLEX_FIELD_IGNORE = new AllowableValue(IGNORE_VALUE, IGNORE_VALUE, "Silently ignore and do not include in row sent to HBase.");
+ protected static final AllowableValue COMPLEX_FIELD_TEXT = new AllowableValue(TEXT_VALUE, TEXT_VALUE, "Use the string representation of the complex field as the value of the given column.");
+
+ protected static final PropertyDescriptor COMPLEX_FIELD_STRATEGY = new PropertyDescriptor.Builder()
+ .name("Complex Field Strategy")
+ .description("Indicates how to handle complex fields, i.e. fields that do not have a single text value.")
+ .expressionLanguageSupported(false)
+ .required(true)
+ .allowableValues(COMPLEX_FIELD_FAIL, COMPLEX_FIELD_WARN, COMPLEX_FIELD_IGNORE, COMPLEX_FIELD_TEXT)
+ .defaultValue(COMPLEX_FIELD_TEXT.getValue())
+ .build();
+
+ @Override
+ public final List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ final List<PropertyDescriptor> properties = new ArrayList<>();
+ properties.add(HBASE_CLIENT_SERVICE);
+ properties.add(TABLE_NAME);
+ properties.add(ROW_ID);
+ properties.add(ROW_FIELD_NAME);
+ properties.add(COLUMN_FAMILY);
+ properties.add(BATCH_SIZE);
+ properties.add(COMPLEX_FIELD_STRATEGY);
+ return properties;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ final Set<Relationship> rels = new HashSet<>();
+ rels.add(REL_SUCCESS);
+ rels.add(REL_FAILURE);
+ return rels;
+ }
+
+ @Override
+ protected Collection<ValidationResult> customValidate(ValidationContext validationContext) {
+ final Collection<ValidationResult> results = new ArrayList<>();
+
+ final String rowId = validationContext.getProperty(ROW_ID).getValue();
+ final String rowFieldName = validationContext.getProperty(ROW_FIELD_NAME).getValue();
+
+ if (StringUtils.isBlank(rowId) && StringUtils.isBlank(rowFieldName)) {
+ results.add(new ValidationResult.Builder()
+ .subject(this.getClass().getSimpleName())
+ .explanation("Row Identifier or Row Identifier Field Name is required")
+ .valid(false)
+ .build());
+ } else if (!StringUtils.isBlank(rowId) && !StringUtils.isBlank(rowFieldName)) {
+ results.add(new ValidationResult.Builder()
+ .subject(this.getClass().getSimpleName())
+ .explanation("Row Identifier and Row Identifier Field Name can not be used together")
+ .valid(false)
+ .build());
+ }
+
+ return results;
+ }
+
+ @Override
+ protected PutFlowFile createPut(final ProcessSession session, final ProcessContext context, final FlowFile flowFile) {
+ final String tableName = context.getProperty(TABLE_NAME).evaluateAttributeExpressions(flowFile).getValue();
+ final String rowId = context.getProperty(ROW_ID).evaluateAttributeExpressions(flowFile).getValue();
+ final String rowFieldName = context.getProperty(ROW_FIELD_NAME).evaluateAttributeExpressions(flowFile).getValue();
+ final String columnFamily = context.getProperty(COLUMN_FAMILY).evaluateAttributeExpressions(flowFile).getValue();
+ final boolean extractRowId = !StringUtils.isBlank(rowFieldName);
+ final String complexFieldStrategy = context.getProperty(COMPLEX_FIELD_STRATEGY).getValue();
+
+ // Parse the JSON document
+ final ObjectMapper mapper = new ObjectMapper();
+ final ObjectHolder<JsonNode> rootNodeRef = new ObjectHolder<>(null);
+ try {
+ session.read(flowFile, new InputStreamCallback() {
+ @Override
+ public void process(final InputStream in) throws IOException {
+ try (final InputStream bufferedIn = new BufferedInputStream(in)) {
+ rootNodeRef.set(mapper.readTree(bufferedIn));
+ }
+ }
+ });
+ } catch (final ProcessException pe) {
+ getLogger().error("Failed to parse {} as JSON due to {}; routing to failure", new Object[]{flowFile, pe.toString()}, pe);
+ return null;
+ }
+
+ final JsonNode rootNode = rootNodeRef.get();
+
+ if (rootNode.isArray()) {
+ getLogger().error("Root node of JSON must be a single document, found array for {}; routing to failure", new Object[]{flowFile});
+ return null;
+ }
+
+ final Collection<PutColumn> columns = new ArrayList<>();
+ final ObjectHolder<String> rowIdHolder = new ObjectHolder<>(null);
+
+ // convert each field/value to a column for the put, skip over nulls and arrays
+ final Iterator<String> fieldNames = rootNode.getFieldNames();
+ while (fieldNames.hasNext()) {
+ final String fieldName = fieldNames.next();
+ final ObjectHolder<String> fieldValueHolder = new ObjectHolder<>(null);
+
+ final JsonNode fieldNode = rootNode.get(fieldName);
+ if (fieldNode.isNull()) {
+ getLogger().debug("Skipping {} because value was null", new Object[]{fieldName});
+ } else if (fieldNode.isValueNode()) {
+ fieldValueHolder.set(fieldNode.asText());
+ } else {
+ // for non-null, non-value nodes, determine what to do based on the handling strategy
+ switch (complexFieldStrategy) {
+ case FAIL_VALUE:
+ getLogger().error("Complex value found for {}; routing to failure", new Object[]{fieldName});
+ return null;
+ case WARN_VALUE:
+ getLogger().warn("Complex value found for {}; skipping", new Object[]{fieldName});
+ break;
+ case TEXT_VALUE:
+ // use toString() here because asText() is only guaranteed to be supported on value nodes
+ // some other types of nodes, like ArrayNode, provide toString implementations
+ fieldValueHolder.set(fieldNode.toString());
+ break;
+ case IGNORE_VALUE:
+ // silently skip
+ break;
+ default:
+ break;
+ }
+ }
+
+ // if we have a field value, then see if this is the row id field, if so store the value for later
+ // otherwise add a new column where the fieldName and fieldValue are the column qualifier and value
+ if (fieldValueHolder.get() != null) {
+ if (extractRowId && fieldName.equals(rowFieldName)) {
+ rowIdHolder.set(fieldValueHolder.get());
+ } else {
+ columns.add(new PutColumn(columnFamily, fieldName, fieldValueHolder.get().getBytes(StandardCharsets.UTF_8)));
+ }
+ }
+ }
+
+ // if we are expecting a field name to use for the row id and the incoming document doesn't have it
+ // log an error message so the user can see what the field names were and return null so it gets routed to failure
+ if (extractRowId && rowIdHolder.get() == null) {
+ final String fieldNameStr = StringUtils.join(rootNode.getFieldNames(), ",");
+ getLogger().error("Row ID field named '{}' not found in field names '{}'; routing to failure", new Object[] {rowFieldName, fieldNameStr});
+ return null;
+ }
+
+ final String putRowId = (extractRowId ? rowIdHolder.get() : rowId);
+ return new PutFlowFile(tableName, putRowId, columns, flowFile);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/40dd8a0a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
index 613515d..6e2af81 100644
--- a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
+++ b/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
@@ -14,4 +14,5 @@
# limitations under the License.
org.apache.nifi.hbase.GetHBase
-org.apache.nifi.hbase.PutHBaseCell
\ No newline at end of file
+org.apache.nifi.hbase.PutHBaseCell
+org.apache.nifi.hbase.PutHBaseJSON
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/40dd8a0a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/test/java/org/apache/nifi/hbase/HBaseTestUtil.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/test/java/org/apache/nifi/hbase/HBaseTestUtil.java b/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/test/java/org/apache/nifi/hbase/HBaseTestUtil.java
new file mode 100644
index 0000000..fc30f73
--- /dev/null
+++ b/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/test/java/org/apache/nifi/hbase/HBaseTestUtil.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hbase;
+
+import org.apache.nifi.hbase.put.PutColumn;
+import org.apache.nifi.hbase.put.PutFlowFile;
+import org.apache.nifi.provenance.ProvenanceEventRecord;
+import org.apache.nifi.provenance.ProvenanceEventType;
+
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertTrue;
+
+public class HBaseTestUtil {
+
+ public static void verifyPut(final String row, final String columnFamily, final Map<String,String> columns, final List<PutFlowFile> puts) {
+ boolean foundPut = false;
+
+ for (final PutFlowFile put : puts) {
+ if (!row.equals(put.getRow())) {
+ continue;
+ }
+
+ if (put.getColumns() == null || put.getColumns().size() != columns.size()) {
+ continue;
+ }
+
+ // start off assuming we have all the columns
+ boolean foundAllColumns = true;
+
+ for (Map.Entry<String, String> entry : columns.entrySet()) {
+ // determine if we have the current expected column
+ boolean foundColumn = false;
+ for (PutColumn putColumn : put.getColumns()) {
+ final String colVal = new String(putColumn.getBuffer(), StandardCharsets.UTF_8);
+ if (columnFamily.equals(putColumn.getColumnFamily()) && entry.getKey().equals(putColumn.getColumnQualifier())
+ && entry.getValue().equals(colVal)) {
+ foundColumn = true;
+ break;
+ }
+ }
+
+ // if we didn't have the current expected column we know we don't have all expected columns
+ if (!foundColumn) {
+ foundAllColumns = false;
+ break;
+ }
+ }
+
+ // if we found all the expected columns this was a match so we can break
+ if (foundAllColumns) {
+ foundPut = true;
+ break;
+ }
+ }
+
+ assertTrue(foundPut);
+ }
+
+ public static void verifyEvent(final List<ProvenanceEventRecord> events, final String uri, final ProvenanceEventType eventType) {
+ boolean foundEvent = false;
+ for (final ProvenanceEventRecord event : events) {
+ if (event.getTransitUri().equals(uri) && event.getEventType().equals(eventType)) {
+ foundEvent = true;
+ break;
+ }
+ }
+ assertTrue(foundEvent);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/40dd8a0a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/test/java/org/apache/nifi/hbase/MockHBaseClientService.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/test/java/org/apache/nifi/hbase/MockHBaseClientService.java b/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/test/java/org/apache/nifi/hbase/MockHBaseClientService.java
index a2abf7e..bca8b4f 100644
--- a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/test/java/org/apache/nifi/hbase/MockHBaseClientService.java
+++ b/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/test/java/org/apache/nifi/hbase/MockHBaseClientService.java
@@ -17,6 +17,7 @@
package org.apache.nifi.hbase;
import org.apache.nifi.controller.AbstractControllerService;
+import org.apache.nifi.hbase.put.PutColumn;
import org.apache.nifi.hbase.put.PutFlowFile;
import org.apache.nifi.hbase.scan.Column;
import org.apache.nifi.hbase.scan.ResultCell;
@@ -33,7 +34,7 @@ import java.util.Map;
public class MockHBaseClientService extends AbstractControllerService implements HBaseClientService {
private Map<String,ResultCell[]> results = new HashMap<>();
- private Map<String, List<PutFlowFile>> puts = new HashMap<>();
+ private Map<String, List<PutFlowFile>> flowFilePuts = new HashMap<>();
private boolean throwException = false;
@Override
@@ -42,7 +43,12 @@ public class MockHBaseClientService extends AbstractControllerService implements
throw new IOException("exception");
}
- this.puts.put(tableName, new ArrayList<>(puts));
+ this.flowFilePuts.put(tableName, new ArrayList<>(puts));
+ }
+
+ @Override
+ public void put(String tableName, String rowId, Collection<PutColumn> columns) throws IOException {
+ throw new UnsupportedOperationException();
}
@Override
@@ -92,8 +98,8 @@ public class MockHBaseClientService extends AbstractControllerService implements
results.put(rowKey, cellArray);
}
- public Map<String, List<PutFlowFile>> getPuts() {
- return puts;
+ public Map<String, List<PutFlowFile>> getFlowFilePuts() {
+ return flowFilePuts;
}
public void setThrowException(boolean throwException) {
http://git-wip-us.apache.org/repos/asf/nifi/blob/40dd8a0a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/test/java/org/apache/nifi/hbase/TestPutHBaseCell.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/test/java/org/apache/nifi/hbase/TestPutHBaseCell.java b/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/test/java/org/apache/nifi/hbase/TestPutHBaseCell.java
index 62fa9a6..0cd8ff7 100644
--- a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/test/java/org/apache/nifi/hbase/TestPutHBaseCell.java
+++ b/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/test/java/org/apache/nifi/hbase/TestPutHBaseCell.java
@@ -16,6 +16,7 @@
*/
package org.apache.nifi.hbase;
+import org.apache.nifi.hbase.put.PutColumn;
import org.apache.nifi.hbase.put.PutFlowFile;
import org.apache.nifi.reporting.InitializationException;
import org.apache.nifi.util.MockFlowFile;
@@ -43,7 +44,7 @@ public class TestPutHBaseCell {
final TestRunner runner = TestRunners.newTestRunner(PutHBaseCell.class);
runner.setProperty(PutHBaseCell.TABLE_NAME, tableName);
- runner.setProperty(PutHBaseCell.ROW, row);
+ runner.setProperty(PutHBaseCell.ROW_ID, row);
runner.setProperty(PutHBaseCell.COLUMN_FAMILY, columnFamily);
runner.setProperty(PutHBaseCell.COLUMN_QUALIFIER, columnQualifier);
runner.setProperty(PutHBaseCell.BATCH_SIZE, "1");
@@ -58,12 +59,14 @@ public class TestPutHBaseCell {
final MockFlowFile outFile = runner.getFlowFilesForRelationship(PutHBaseCell.REL_SUCCESS).get(0);
outFile.assertContentEquals(content);
- assertNotNull(hBaseClient.getPuts());
- assertEquals(1, hBaseClient.getPuts().size());
+ assertNotNull(hBaseClient.getFlowFilePuts());
+ assertEquals(1, hBaseClient.getFlowFilePuts().size());
- List<PutFlowFile> puts = hBaseClient.getPuts().get(tableName);
+ List<PutFlowFile> puts = hBaseClient.getFlowFilePuts().get(tableName);
assertEquals(1, puts.size());
verifyPut(row, columnFamily, columnQualifier, content, puts.get(0));
+
+ assertEquals(1, runner.getProvenanceEvents().size());
}
@Test
@@ -89,12 +92,14 @@ public class TestPutHBaseCell {
final MockFlowFile outFile = runner.getFlowFilesForRelationship(PutHBaseCell.REL_SUCCESS).get(0);
outFile.assertContentEquals(content);
- assertNotNull(hBaseClient.getPuts());
- assertEquals(1, hBaseClient.getPuts().size());
+ assertNotNull(hBaseClient.getFlowFilePuts());
+ assertEquals(1, hBaseClient.getFlowFilePuts().size());
- List<PutFlowFile> puts = hBaseClient.getPuts().get(tableName);
+ List<PutFlowFile> puts = hBaseClient.getFlowFilePuts().get(tableName);
assertEquals(1, puts.size());
verifyPut(row, columnFamily, columnQualifier, content, puts.get(0));
+
+ assertEquals(1, runner.getProvenanceEvents().size());
}
@Test
@@ -115,7 +120,9 @@ public class TestPutHBaseCell {
runner.run();
runner.assertTransferCount(PutHBaseCell.REL_SUCCESS, 0);
- runner.assertTransferCount(PutHBaseCell.FAILURE, 1);
+ runner.assertTransferCount(PutHBaseCell.REL_FAILURE, 1);
+
+ assertEquals(0, runner.getProvenanceEvents().size());
}
@Test
@@ -142,7 +149,9 @@ public class TestPutHBaseCell {
runner.run();
runner.assertTransferCount(PutHBaseCell.REL_SUCCESS, 1);
- runner.assertTransferCount(PutHBaseCell.FAILURE, 1);
+ runner.assertTransferCount(PutHBaseCell.REL_FAILURE, 1);
+
+ assertEquals(1, runner.getProvenanceEvents().size());
}
@Test
@@ -171,13 +180,15 @@ public class TestPutHBaseCell {
final MockFlowFile outFile = runner.getFlowFilesForRelationship(PutHBaseCell.REL_SUCCESS).get(0);
outFile.assertContentEquals(content1);
- assertNotNull(hBaseClient.getPuts());
- assertEquals(1, hBaseClient.getPuts().size());
+ assertNotNull(hBaseClient.getFlowFilePuts());
+ assertEquals(1, hBaseClient.getFlowFilePuts().size());
- List<PutFlowFile> puts = hBaseClient.getPuts().get(tableName);
+ List<PutFlowFile> puts = hBaseClient.getFlowFilePuts().get(tableName);
assertEquals(2, puts.size());
verifyPut(row1, columnFamily, columnQualifier, content1, puts.get(0));
verifyPut(row2, columnFamily, columnQualifier, content2, puts.get(1));
+
+ assertEquals(2, runner.getProvenanceEvents().size());
}
@Test
@@ -202,7 +213,9 @@ public class TestPutHBaseCell {
runner.enqueue(content2.getBytes("UTF-8"), attributes2);
runner.run();
- runner.assertAllFlowFilesTransferred(PutHBaseCell.FAILURE, 2);
+ runner.assertAllFlowFilesTransferred(PutHBaseCell.REL_FAILURE, 2);
+
+ assertEquals(0, runner.getProvenanceEvents().size());
}
@Test
@@ -229,13 +242,15 @@ public class TestPutHBaseCell {
final MockFlowFile outFile = runner.getFlowFilesForRelationship(PutHBaseCell.REL_SUCCESS).get(0);
outFile.assertContentEquals(content1);
- assertNotNull(hBaseClient.getPuts());
- assertEquals(1, hBaseClient.getPuts().size());
+ assertNotNull(hBaseClient.getFlowFilePuts());
+ assertEquals(1, hBaseClient.getFlowFilePuts().size());
- List<PutFlowFile> puts = hBaseClient.getPuts().get(tableName);
+ List<PutFlowFile> puts = hBaseClient.getFlowFilePuts().get(tableName);
assertEquals(2, puts.size());
verifyPut(row, columnFamily, columnQualifier, content1, puts.get(0));
verifyPut(row, columnFamily, columnQualifier, content2, puts.get(1));
+
+ assertEquals(2, runner.getProvenanceEvents().size());
}
private Map<String, String> getAtrributeMapWithEL(String tableName, String row, String columnFamily, String columnQualifier) {
@@ -250,7 +265,7 @@ public class TestPutHBaseCell {
private TestRunner getTestRunnerWithEL(PutHBaseCell proc) {
final TestRunner runner = TestRunners.newTestRunner(proc);
runner.setProperty(PutHBaseCell.TABLE_NAME, "${hbase.tableName}");
- runner.setProperty(PutHBaseCell.ROW, "${hbase.row}");
+ runner.setProperty(PutHBaseCell.ROW_ID, "${hbase.row}");
runner.setProperty(PutHBaseCell.COLUMN_FAMILY, "${hbase.columnFamily}");
runner.setProperty(PutHBaseCell.COLUMN_QUALIFIER, "${hbase.columnQualifier}");
return runner;
@@ -266,9 +281,14 @@ public class TestPutHBaseCell {
private void verifyPut(String row, String columnFamily, String columnQualifier, String content, PutFlowFile put) {
assertEquals(row, put.getRow());
- assertEquals(columnFamily, put.getColumnFamily());
- assertEquals(columnQualifier, put.getColumnQualifier());
- assertEquals(content, new String(put.getBuffer(), StandardCharsets.UTF_8));
+
+ assertNotNull(put.getColumns());
+ assertEquals(1, put.getColumns().size());
+
+ final PutColumn column = put.getColumns().iterator().next();
+ assertEquals(columnFamily, column.getColumnFamily());
+ assertEquals(columnQualifier, column.getColumnQualifier());
+ assertEquals(content, new String(column.getBuffer(), StandardCharsets.UTF_8));
}
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/40dd8a0a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/test/java/org/apache/nifi/hbase/TestPutHBaseJSON.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/test/java/org/apache/nifi/hbase/TestPutHBaseJSON.java b/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/test/java/org/apache/nifi/hbase/TestPutHBaseJSON.java
new file mode 100644
index 0000000..7b59919
--- /dev/null
+++ b/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/test/java/org/apache/nifi/hbase/TestPutHBaseJSON.java
@@ -0,0 +1,423 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hbase;
+
+import org.apache.nifi.hbase.put.PutFlowFile;
+import org.apache.nifi.provenance.ProvenanceEventRecord;
+import org.apache.nifi.provenance.ProvenanceEventType;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+public class TestPutHBaseJSON {
+
+ public static final String DEFAULT_TABLE_NAME = "nifi";
+ public static final String DEFAULT_ROW = "row1";
+ public static final String DEFAULT_COLUMN_FAMILY = "family1";
+
+ @Test
+ public void testCustomValidate() throws InitializationException {
+ // missing row id and row id field name should be invalid
+ TestRunner runner = getTestRunner(DEFAULT_TABLE_NAME, DEFAULT_COLUMN_FAMILY, "1");
+ getHBaseClientService(runner);
+ runner.assertNotValid();
+
+ // setting both properties should still be invalid
+ runner = getTestRunner(DEFAULT_TABLE_NAME, DEFAULT_COLUMN_FAMILY, "1");
+ getHBaseClientService(runner);
+ runner.setProperty(PutHBaseJSON.ROW_ID, "rowId");
+ runner.setProperty(PutHBaseJSON.ROW_FIELD_NAME, "rowFieldName");
+ runner.assertNotValid();
+
+ // only a row id field name should make it valid
+ runner = getTestRunner(DEFAULT_TABLE_NAME, DEFAULT_COLUMN_FAMILY, "1");
+ getHBaseClientService(runner);
+ runner.setProperty(PutHBaseJSON.ROW_FIELD_NAME, "rowFieldName");
+ runner.assertValid();
+
+ // only a row id should make it valid
+ runner = getTestRunner(DEFAULT_TABLE_NAME, DEFAULT_COLUMN_FAMILY, "1");
+ getHBaseClientService(runner);
+ runner.setProperty(PutHBaseJSON.ROW_ID, "rowId");
+ runner.assertValid();
+ }
+
+ @Test
+ public void testSingleJsonDocAndProvidedRowId() throws IOException, InitializationException {
+ final TestRunner runner = getTestRunner(DEFAULT_TABLE_NAME, DEFAULT_COLUMN_FAMILY, "1");
+ final MockHBaseClientService hBaseClient = getHBaseClientService(runner);
+ runner.setProperty(PutHBaseJSON.ROW_ID, DEFAULT_ROW);
+
+ final String content = "{ \"field1\" : \"value1\", \"field2\" : \"value2\" }";
+ runner.enqueue(content.getBytes("UTF-8"));
+ runner.run();
+ runner.assertAllFlowFilesTransferred(PutHBaseCell.REL_SUCCESS);
+
+ final MockFlowFile outFile = runner.getFlowFilesForRelationship(PutHBaseCell.REL_SUCCESS).get(0);
+ outFile.assertContentEquals(content);
+
+ assertNotNull(hBaseClient.getFlowFilePuts());
+ assertEquals(1, hBaseClient.getFlowFilePuts().size());
+
+ final List<PutFlowFile> puts = hBaseClient.getFlowFilePuts().get(DEFAULT_TABLE_NAME);
+ assertEquals(1, puts.size());
+
+ final Map<String,String> expectedColumns = new HashMap<>();
+ expectedColumns.put("field1", "value1");
+ expectedColumns.put("field2", "value2");
+ HBaseTestUtil.verifyPut(DEFAULT_ROW, DEFAULT_COLUMN_FAMILY, expectedColumns, puts);
+
+ final List<ProvenanceEventRecord> events = runner.getProvenanceEvents();
+ assertEquals(1, events.size());
+
+ final ProvenanceEventRecord event = events.get(0);
+ assertEquals("hbase://" + DEFAULT_TABLE_NAME + "/" + DEFAULT_ROW, event.getTransitUri());
+ }
+
+ @Test
+ public void testSingJsonDocAndExtractedRowId() throws IOException, InitializationException {
+ final TestRunner runner = getTestRunner(DEFAULT_TABLE_NAME, DEFAULT_COLUMN_FAMILY, "1");
+ final MockHBaseClientService hBaseClient = getHBaseClientService(runner);
+ runner.setProperty(PutHBaseJSON.ROW_FIELD_NAME, "rowField");
+
+ final String content = "{ \"rowField\" : \"myRowId\", \"field1\" : \"value1\", \"field2\" : \"value2\" }";
+ runner.enqueue(content.getBytes(StandardCharsets.UTF_8));
+ runner.run();
+ runner.assertAllFlowFilesTransferred(PutHBaseCell.REL_SUCCESS);
+
+ final MockFlowFile outFile = runner.getFlowFilesForRelationship(PutHBaseCell.REL_SUCCESS).get(0);
+ outFile.assertContentEquals(content);
+
+ assertNotNull(hBaseClient.getFlowFilePuts());
+ assertEquals(1, hBaseClient.getFlowFilePuts().size());
+
+ final List<PutFlowFile> puts = hBaseClient.getFlowFilePuts().get(DEFAULT_TABLE_NAME);
+ assertEquals(1, puts.size());
+
+ // should be a put with row id of myRowId, and rowField shouldn't end up in the columns
+ final Map<String,String> expectedColumns1 = new HashMap<>();
+ expectedColumns1.put("field1", "value1");
+ expectedColumns1.put("field2", "value2");
+ HBaseTestUtil.verifyPut("myRowId", DEFAULT_COLUMN_FAMILY, expectedColumns1, puts);
+
+ final List<ProvenanceEventRecord> events = runner.getProvenanceEvents();
+ assertEquals(1, events.size());
+ HBaseTestUtil.verifyEvent(runner.getProvenanceEvents(), "hbase://" + DEFAULT_TABLE_NAME + "/myRowId", ProvenanceEventType.SEND);
+ }
+
+ @Test
+ public void testSingJsonDocAndExtractedRowIdMissingField() throws IOException, InitializationException {
+ final TestRunner runner = getTestRunner(DEFAULT_TABLE_NAME, DEFAULT_COLUMN_FAMILY, "1");
+ final MockHBaseClientService hBaseClient = getHBaseClientService(runner);
+ runner.setProperty(PutHBaseJSON.ROW_FIELD_NAME, "rowField");
+
+ final String content = "{ \"field1\" : \"value1\", \"field2\" : \"value2\" }";
+ runner.enqueue(content.getBytes(StandardCharsets.UTF_8));
+ runner.run();
+ runner.assertAllFlowFilesTransferred(PutHBaseCell.REL_FAILURE, 1);
+
+ final MockFlowFile outFile = runner.getFlowFilesForRelationship(PutHBaseCell.REL_FAILURE).get(0);
+ outFile.assertContentEquals(content);
+
+ // should be no provenance events
+ assertEquals(0, runner.getProvenanceEvents().size());
+
+ // no puts should have made it to the client
+ assertEquals(0, hBaseClient.getFlowFilePuts().size());
+ }
+
+ @Test
+ public void testMultipleJsonDocsRouteToFailure() throws IOException, InitializationException {
+ final TestRunner runner = getTestRunner(DEFAULT_TABLE_NAME, DEFAULT_COLUMN_FAMILY, "1");
+ final MockHBaseClientService hBaseClient = getHBaseClientService(runner);
+ runner.setProperty(PutHBaseJSON.ROW_ID, DEFAULT_ROW);
+
+ final String content1 = "{ \"field1\" : \"value1\", \"field2\" : \"value2\" }";
+ final String content2 = "{ \"field3\" : \"value3\", \"field4\" : \"value4\" }";
+ final String content = "[ " + content1 + " , " + content2 + " ]";
+
+ runner.enqueue(content.getBytes(StandardCharsets.UTF_8));
+ runner.run();
+ runner.assertAllFlowFilesTransferred(PutHBaseCell.REL_FAILURE, 1);
+
+ final MockFlowFile outFile = runner.getFlowFilesForRelationship(PutHBaseCell.REL_FAILURE).get(0);
+ outFile.assertContentEquals(content);
+
+ // should be no provenance events
+ assertEquals(0, runner.getProvenanceEvents().size());
+
+ // no puts should have made it to the client
+ assertEquals(0, hBaseClient.getFlowFilePuts().size());
+ }
+
+ @Test
+ public void testELWithProvidedRowId() throws IOException, InitializationException {
+ final TestRunner runner = getTestRunner("${hbase.table}", "${hbase.colFamily}", "1");
+ final MockHBaseClientService hBaseClient = getHBaseClientService(runner);
+ runner.setProperty(PutHBaseJSON.ROW_ID, "${hbase.rowId}");
+
+ final Map<String,String> attributes = new HashMap<>();
+ attributes.put("hbase.table", "myTable");
+ attributes.put("hbase.colFamily", "myColFamily");
+ attributes.put("hbase.rowId", "myRowId");
+
+ final String content = "{ \"field1\" : \"value1\", \"field2\" : \"value2\" }";
+ runner.enqueue(content.getBytes("UTF-8"), attributes);
+ runner.run();
+ runner.assertAllFlowFilesTransferred(PutHBaseCell.REL_SUCCESS);
+
+ final MockFlowFile outFile = runner.getFlowFilesForRelationship(PutHBaseCell.REL_SUCCESS).get(0);
+ outFile.assertContentEquals(content);
+
+ assertNotNull(hBaseClient.getFlowFilePuts());
+ assertEquals(1, hBaseClient.getFlowFilePuts().size());
+
+ final List<PutFlowFile> puts = hBaseClient.getFlowFilePuts().get("myTable");
+ assertEquals(1, puts.size());
+
+ final Map<String,String> expectedColumns = new HashMap<>();
+ expectedColumns.put("field1", "value1");
+ expectedColumns.put("field2", "value2");
+ HBaseTestUtil.verifyPut("myRowId", "myColFamily", expectedColumns, puts);
+
+ final List<ProvenanceEventRecord> events = runner.getProvenanceEvents();
+ assertEquals(1, events.size());
+ HBaseTestUtil.verifyEvent(runner.getProvenanceEvents(), "hbase://myTable/myRowId", ProvenanceEventType.SEND);
+ }
+
+ @Test
+ public void testELWithExtractedRowId() throws IOException, InitializationException {
+ final TestRunner runner = getTestRunner("${hbase.table}", "${hbase.colFamily}", "1");
+ final MockHBaseClientService hBaseClient = getHBaseClientService(runner);
+ runner.setProperty(PutHBaseJSON.ROW_FIELD_NAME, "${hbase.rowField}");
+
+ final Map<String,String> attributes = new HashMap<>();
+ attributes.put("hbase.table", "myTable");
+ attributes.put("hbase.colFamily", "myColFamily");
+ attributes.put("hbase.rowField", "field1");
+
+ final String content = "{ \"field1\" : \"value1\", \"field2\" : \"value2\" }";
+ runner.enqueue(content.getBytes("UTF-8"), attributes);
+ runner.run();
+ runner.assertAllFlowFilesTransferred(PutHBaseCell.REL_SUCCESS);
+
+ final MockFlowFile outFile = runner.getFlowFilesForRelationship(PutHBaseCell.REL_SUCCESS).get(0);
+ outFile.assertContentEquals(content);
+
+ assertNotNull(hBaseClient.getFlowFilePuts());
+ assertEquals(1, hBaseClient.getFlowFilePuts().size());
+
+ final List<PutFlowFile> puts = hBaseClient.getFlowFilePuts().get("myTable");
+ assertEquals(1, puts.size());
+
+ final Map<String,String> expectedColumns = new HashMap<>();
+ expectedColumns.put("field2", "value2");
+ HBaseTestUtil.verifyPut("value1", "myColFamily", expectedColumns, puts);
+
+ final List<ProvenanceEventRecord> events = runner.getProvenanceEvents();
+ assertEquals(1, events.size());
+ HBaseTestUtil.verifyEvent(runner.getProvenanceEvents(), "hbase://myTable/value1", ProvenanceEventType.SEND);
+ }
+
+ @Test
+ public void testNullAndArrayElementsWithWarnStrategy() throws InitializationException {
+ final TestRunner runner = getTestRunner(DEFAULT_TABLE_NAME, DEFAULT_COLUMN_FAMILY, "1");
+ final MockHBaseClientService hBaseClient = getHBaseClientService(runner);
+ runner.setProperty(PutHBaseJSON.ROW_ID, DEFAULT_ROW);
+ runner.setProperty(PutHBaseJSON.COMPLEX_FIELD_STRATEGY, PutHBaseJSON.COMPLEX_FIELD_WARN.getValue());
+
+ // should route to success because there is at least one valid field
+ final String content = "{ \"field1\" : [{ \"child_field1\" : \"child_value1\" }], \"field2\" : \"value2\", \"field3\" : null }";
+ runner.enqueue(content.getBytes(StandardCharsets.UTF_8));
+ runner.run();
+ runner.assertAllFlowFilesTransferred(PutHBaseCell.REL_SUCCESS);
+
+ assertNotNull(hBaseClient.getFlowFilePuts());
+ assertEquals(1, hBaseClient.getFlowFilePuts().size());
+
+ final List<PutFlowFile> puts = hBaseClient.getFlowFilePuts().get(DEFAULT_TABLE_NAME);
+ assertEquals(1, puts.size());
+
+ // should have skipped field1 and field3
+ final Map<String,String> expectedColumns = new HashMap<>();
+ expectedColumns.put("field2", "value2");
+ HBaseTestUtil.verifyPut(DEFAULT_ROW, DEFAULT_COLUMN_FAMILY, expectedColumns, puts);
+ }
+
+ @Test
+ public void testNullAndArrayElementsWithIgnoreStrategy() throws InitializationException {
+ final TestRunner runner = getTestRunner(DEFAULT_TABLE_NAME, DEFAULT_COLUMN_FAMILY, "1");
+ final MockHBaseClientService hBaseClient = getHBaseClientService(runner);
+ runner.setProperty(PutHBaseJSON.ROW_ID, DEFAULT_ROW);
+ runner.setProperty(PutHBaseJSON.COMPLEX_FIELD_STRATEGY, PutHBaseJSON.COMPLEX_FIELD_IGNORE.getValue());
+
+ // should route to success because there is at least one valid field
+ final String content = "{ \"field1\" : [{ \"child_field1\" : \"child_value1\" }], \"field2\" : \"value2\", \"field3\" : null }";
+ runner.enqueue(content.getBytes(StandardCharsets.UTF_8));
+ runner.run();
+ runner.assertAllFlowFilesTransferred(PutHBaseCell.REL_SUCCESS);
+
+ assertNotNull(hBaseClient.getFlowFilePuts());
+ assertEquals(1, hBaseClient.getFlowFilePuts().size());
+
+ final List<PutFlowFile> puts = hBaseClient.getFlowFilePuts().get(DEFAULT_TABLE_NAME);
+ assertEquals(1, puts.size());
+
+ // should have skipped field1 and field3
+ final Map<String,String> expectedColumns = new HashMap<>();
+ expectedColumns.put("field2", "value2");
+ HBaseTestUtil.verifyPut(DEFAULT_ROW, DEFAULT_COLUMN_FAMILY, expectedColumns, puts);
+ }
+
+ @Test
+ public void testNullAndArrayElementsWithFailureStrategy() throws InitializationException {
+ final TestRunner runner = getTestRunner(DEFAULT_TABLE_NAME, DEFAULT_COLUMN_FAMILY, "1");
+ final MockHBaseClientService hBaseClient = getHBaseClientService(runner);
+ runner.setProperty(PutHBaseJSON.ROW_ID, DEFAULT_ROW);
+ runner.setProperty(PutHBaseJSON.COMPLEX_FIELD_STRATEGY, PutHBaseJSON.COMPLEX_FIELD_FAIL.getValue());
+
+ // should route to success because there is at least one valid field
+ final String content = "{ \"field1\" : [{ \"child_field1\" : \"child_value1\" }], \"field2\" : \"value2\", \"field3\" : null }";
+ runner.enqueue(content.getBytes(StandardCharsets.UTF_8));
+ runner.run();
+ runner.assertAllFlowFilesTransferred(PutHBaseCell.REL_FAILURE, 1);
+
+ final MockFlowFile outFile = runner.getFlowFilesForRelationship(PutHBaseCell.REL_FAILURE).get(0);
+ outFile.assertContentEquals(content);
+
+ // should be no provenance events
+ assertEquals(0, runner.getProvenanceEvents().size());
+
+ // no puts should have made it to the client
+ assertEquals(0, hBaseClient.getFlowFilePuts().size());
+ }
+
+ @Test
+ public void testNullAndArrayElementsWithTextStrategy() throws InitializationException {
+ final TestRunner runner = getTestRunner(DEFAULT_TABLE_NAME, DEFAULT_COLUMN_FAMILY, "1");
+ final MockHBaseClientService hBaseClient = getHBaseClientService(runner);
+ runner.setProperty(PutHBaseJSON.ROW_ID, DEFAULT_ROW);
+ runner.setProperty(PutHBaseJSON.COMPLEX_FIELD_STRATEGY, PutHBaseJSON.COMPLEX_FIELD_TEXT.getValue());
+
+ // should route to success because there is at least one valid field
+ final String content = "{ \"field1\" : [{ \"child_field1\" : \"child_value1\" }], \"field2\" : \"value2\", \"field3\" : null }";
+ runner.enqueue(content.getBytes(StandardCharsets.UTF_8));
+ runner.run();
+ runner.assertAllFlowFilesTransferred(PutHBaseCell.REL_SUCCESS);
+
+ assertNotNull(hBaseClient.getFlowFilePuts());
+ assertEquals(1, hBaseClient.getFlowFilePuts().size());
+
+ final List<PutFlowFile> puts = hBaseClient.getFlowFilePuts().get(DEFAULT_TABLE_NAME);
+ assertEquals(1, puts.size());
+
+ // should have skipped field1 and field3
+ final Map<String,String> expectedColumns = new HashMap<>();
+ expectedColumns.put("field1", "[{\"child_field1\":\"child_value1\"}]");
+ expectedColumns.put("field2", "value2");
+ HBaseTestUtil.verifyPut(DEFAULT_ROW, DEFAULT_COLUMN_FAMILY, expectedColumns, puts);
+ }
+
+ @Test
+ public void testNestedDocWithTextStrategy() throws InitializationException {
+ final TestRunner runner = getTestRunner(DEFAULT_TABLE_NAME, DEFAULT_COLUMN_FAMILY, "1");
+ final MockHBaseClientService hBaseClient = getHBaseClientService(runner);
+ runner.setProperty(PutHBaseJSON.ROW_ID, DEFAULT_ROW);
+ runner.setProperty(PutHBaseJSON.COMPLEX_FIELD_STRATEGY, PutHBaseJSON.COMPLEX_FIELD_TEXT.getValue());
+
+ // should route to success because there is at least one valid field
+ final String content = "{ \"field1\" : { \"child_field1\" : \"child_value1\" }, \"field2\" : \"value2\", \"field3\" : null }";
+ runner.enqueue(content.getBytes(StandardCharsets.UTF_8));
+ runner.run();
+ runner.assertAllFlowFilesTransferred(PutHBaseCell.REL_SUCCESS);
+
+ assertNotNull(hBaseClient.getFlowFilePuts());
+ assertEquals(1, hBaseClient.getFlowFilePuts().size());
+
+ final List<PutFlowFile> puts = hBaseClient.getFlowFilePuts().get(DEFAULT_TABLE_NAME);
+ assertEquals(1, puts.size());
+
+ // should have skipped field1 and field3
+ final Map<String,String> expectedColumns = new HashMap<>();
+ expectedColumns.put("field1", "{\"child_field1\":\"child_value1\"}");
+ expectedColumns.put("field2", "value2");
+ HBaseTestUtil.verifyPut(DEFAULT_ROW, DEFAULT_COLUMN_FAMILY, expectedColumns, puts);
+ }
+
+ @Test
+ public void testAllElementsAreNullOrArrays() throws InitializationException {
+ final TestRunner runner = getTestRunner(DEFAULT_TABLE_NAME, DEFAULT_COLUMN_FAMILY, "1");
+ final MockHBaseClientService hBaseClient = getHBaseClientService(runner);
+ runner.setProperty(PutHBaseJSON.ROW_ID, DEFAULT_ROW);
+ runner.setProperty(PutHBaseJSON.COMPLEX_FIELD_STRATEGY, PutHBaseJSON.COMPLEX_FIELD_WARN.getValue());
+
+ // should route to failure since it would produce a put with no columns
+ final String content = "{ \"field1\" : [{ \"child_field1\" : \"child_value1\" }], \"field2\" : null }";
+ runner.enqueue(content.getBytes(StandardCharsets.UTF_8));
+ runner.run();
+ runner.assertAllFlowFilesTransferred(PutHBaseCell.REL_FAILURE, 1);
+
+ final MockFlowFile outFile = runner.getFlowFilesForRelationship(PutHBaseCell.REL_FAILURE).get(0);
+ outFile.assertContentEquals(content);
+
+ // should be no provenance events
+ assertEquals(0, runner.getProvenanceEvents().size());
+
+ // no puts should have made it to the client
+ assertEquals(0, hBaseClient.getFlowFilePuts().size());
+ }
+
+ @Test
+ public void testInvalidJson() throws InitializationException {
+ final TestRunner runner = getTestRunner(DEFAULT_TABLE_NAME, DEFAULT_COLUMN_FAMILY, "1");
+ getHBaseClientService(runner);
+ runner.setProperty(PutHBaseJSON.ROW_ID, DEFAULT_ROW);
+
+ final String content = "NOT JSON";
+ runner.enqueue(content.getBytes(StandardCharsets.UTF_8));
+ runner.run();
+ runner.assertAllFlowFilesTransferred(PutHBaseCell.REL_FAILURE, 1);
+ }
+
+ private TestRunner getTestRunner(String table, String columnFamily, String batchSize) {
+ final TestRunner runner = TestRunners.newTestRunner(PutHBaseJSON.class);
+ runner.setProperty(PutHBaseJSON.TABLE_NAME, table);
+ runner.setProperty(PutHBaseJSON.COLUMN_FAMILY, columnFamily);
+ runner.setProperty(PutHBaseJSON.BATCH_SIZE, batchSize);
+ return runner;
+ }
+
+ private MockHBaseClientService getHBaseClientService(final TestRunner runner) throws InitializationException {
+ final MockHBaseClientService hBaseClient = new MockHBaseClientService();
+ runner.addControllerService("hbaseClient", hBaseClient);
+ runner.enableControllerService(hBaseClient);
+ runner.setProperty(PutHBaseCell.HBASE_CLIENT_SERVICE, "hbaseClient");
+ return hBaseClient;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/40dd8a0a/nifi-nar-bundles/nifi-standard-services/nifi-hbase-client-service-api/src/main/java/org/apache/nifi/hbase/HBaseClientService.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-hbase-client-service-api/src/main/java/org/apache/nifi/hbase/HBaseClientService.java b/nifi-nar-bundles/nifi-standard-services/nifi-hbase-client-service-api/src/main/java/org/apache/nifi/hbase/HBaseClientService.java
index 9ff2c46..79eef92 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-hbase-client-service-api/src/main/java/org/apache/nifi/hbase/HBaseClientService.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-hbase-client-service-api/src/main/java/org/apache/nifi/hbase/HBaseClientService.java
@@ -20,6 +20,7 @@ import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.controller.ControllerService;
+import org.apache.nifi.hbase.put.PutColumn;
import org.apache.nifi.hbase.put.PutFlowFile;
import org.apache.nifi.hbase.scan.Column;
import org.apache.nifi.hbase.scan.ResultHandler;
@@ -74,6 +75,16 @@ public interface HBaseClientService extends ControllerService {
void put(String tableName, Collection<PutFlowFile> puts) throws IOException;
/**
+ * Puts the given row to HBase with the provided columns.
+ *
+ * @param tableName the name of an HBase table
+ * @param rowId the id of the row to put
+ * @param columns the columns of the row to put
+ * @throws IOException thrown when there are communication errors with HBase
+ */
+ void put(String tableName, String rowId, Collection<PutColumn> columns) throws IOException;
+
+ /**
* Scans the given table using the optional filter criteria and passing each result to the provided handler.
*
* @param tableName the name of an HBase table to scan
http://git-wip-us.apache.org/repos/asf/nifi/blob/40dd8a0a/nifi-nar-bundles/nifi-standard-services/nifi-hbase-client-service-api/src/main/java/org/apache/nifi/hbase/put/PutColumn.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-hbase-client-service-api/src/main/java/org/apache/nifi/hbase/put/PutColumn.java b/nifi-nar-bundles/nifi-standard-services/nifi-hbase-client-service-api/src/main/java/org/apache/nifi/hbase/put/PutColumn.java
new file mode 100644
index 0000000..0971f94
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-hbase-client-service-api/src/main/java/org/apache/nifi/hbase/put/PutColumn.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.hbase.put;
+
+/**
+ * Encapsulates the information for one column of a put operation.
+ */
+public class PutColumn {
+
+ private final String columnFamily;
+ private final String columnQualifier;
+ private final byte[] buffer;
+
+
+ public PutColumn(final String columnFamily, final String columnQualifier, final byte[] buffer) {
+ this.columnFamily = columnFamily;
+ this.columnQualifier = columnQualifier;
+ this.buffer = buffer;
+ }
+
+ public String getColumnFamily() {
+ return columnFamily;
+ }
+
+ public String getColumnQualifier() {
+ return columnQualifier;
+ }
+
+ public byte[] getBuffer() {
+ return buffer;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/40dd8a0a/nifi-nar-bundles/nifi-standard-services/nifi-hbase-client-service-api/src/main/java/org/apache/nifi/hbase/put/PutFlowFile.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-hbase-client-service-api/src/main/java/org/apache/nifi/hbase/put/PutFlowFile.java b/nifi-nar-bundles/nifi-standard-services/nifi-hbase-client-service-api/src/main/java/org/apache/nifi/hbase/put/PutFlowFile.java
index ed6319e..a97e3a4 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-hbase-client-service-api/src/main/java/org/apache/nifi/hbase/put/PutFlowFile.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-hbase-client-service-api/src/main/java/org/apache/nifi/hbase/put/PutFlowFile.java
@@ -16,8 +16,11 @@
*/
package org.apache.nifi.hbase.put;
+import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.flowfile.FlowFile;
+import java.util.Collection;
+
/**
* Wrapper to encapsulate all of the information for the Put along with the FlowFile.
*/
@@ -25,18 +28,13 @@ public class PutFlowFile {
private final String tableName;
private final String row;
- private final String columnFamily;
- private final String columnQualifier;
- private final byte[] buffer;
+ private final Collection<PutColumn> columns;
private final FlowFile flowFile;
- public PutFlowFile(String tableName, String row, String columnFamily, String columnQualifier,
- byte[] buffer, FlowFile flowFile) {
+ public PutFlowFile(String tableName, String row, Collection<PutColumn> columns, FlowFile flowFile) {
this.tableName = tableName;
this.row = row;
- this.columnFamily = columnFamily;
- this.columnQualifier = columnQualifier;
- this.buffer = buffer;
+ this.columns = columns;
this.flowFile = flowFile;
}
@@ -48,20 +46,26 @@ public class PutFlowFile {
return row;
}
- public String getColumnFamily() {
- return columnFamily;
+ public Collection<PutColumn> getColumns() {
+ return columns;
}
- public String getColumnQualifier() {
- return columnQualifier;
+ public FlowFile getFlowFile() {
+ return flowFile;
}
- public byte[] getBuffer() {
- return buffer;
- }
+ public boolean isValid() {
+ if (StringUtils.isBlank(tableName) || StringUtils.isBlank(row) || flowFile == null || columns == null || columns.isEmpty()) {
+ return false;
+ }
- public FlowFile getFlowFile() {
- return flowFile;
+ for (PutColumn column : columns) {
+ if (StringUtils.isBlank(column.getColumnQualifier()) || StringUtils.isBlank(column.getColumnFamily()) || column.getBuffer() == null) {
+ return false;
+ }
+ }
+
+ return true;
}
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/40dd8a0a/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/main/java/org/apache/nifi/hbase/HBase_1_1_2_ClientService.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/main/java/org/apache/nifi/hbase/HBase_1_1_2_ClientService.java b/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/main/java/org/apache/nifi/hbase/HBase_1_1_2_ClientService.java
index 42590c2..b207191 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/main/java/org/apache/nifi/hbase/HBase_1_1_2_ClientService.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/main/java/org/apache/nifi/hbase/HBase_1_1_2_ClientService.java
@@ -43,6 +43,7 @@ import org.apache.nifi.components.ValidationResult;
import org.apache.nifi.controller.AbstractControllerService;
import org.apache.nifi.controller.ConfigurationContext;
import org.apache.nifi.controller.ControllerServiceInitializationContext;
+import org.apache.nifi.hbase.put.PutColumn;
import org.apache.nifi.hbase.put.PutFlowFile;
import org.apache.nifi.hbase.scan.Column;
import org.apache.nifi.hbase.scan.ResultCell;
@@ -195,9 +196,13 @@ public class HBase_1_1_2_ClientService extends AbstractControllerService impleme
put = new Put(putFlowFile.getRow().getBytes(StandardCharsets.UTF_8));
rowPuts.put(putFlowFile.getRow(), put);
}
- put.addColumn(putFlowFile.getColumnFamily().getBytes(StandardCharsets.UTF_8),
- putFlowFile.getColumnQualifier().getBytes(StandardCharsets.UTF_8),
- putFlowFile.getBuffer());
+
+ for (final PutColumn column : putFlowFile.getColumns()) {
+ put.addColumn(
+ column.getColumnFamily().getBytes(StandardCharsets.UTF_8),
+ column.getColumnQualifier().getBytes(StandardCharsets.UTF_8),
+ column.getBuffer());
+ }
}
table.put(new ArrayList<>(rowPuts.values()));
@@ -205,6 +210,20 @@ public class HBase_1_1_2_ClientService extends AbstractControllerService impleme
}
@Override
+ public void put(final String tableName, final String rowId, final Collection<PutColumn> columns) throws IOException {
+ try (final Table table = connection.getTable(TableName.valueOf(tableName))) {
+ Put put = new Put(rowId.getBytes(StandardCharsets.UTF_8));
+ for (final PutColumn column : columns) {
+ put.addColumn(
+ column.getColumnFamily().getBytes(StandardCharsets.UTF_8),
+ column.getColumnQualifier().getBytes(StandardCharsets.UTF_8),
+ column.getBuffer());
+ }
+ table.put(put);
+ }
+ }
+
+ @Override
public void scan(final String tableName, final Collection<Column> columns, final String filterExpression, final long minTime, final ResultHandler handler)
throws IOException {
[39/50] [abbrv] nifi git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/nifi
Posted by mc...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/nifi
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/4e2c94d6
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/4e2c94d6
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/4e2c94d6
Branch: refs/heads/NIFI-655
Commit: 4e2c94d6590e57b3aa7fa4a9180626a48b9ae2e2
Parents: 2516b1d b74617b
Author: Mark Payne <ma...@hotmail.com>
Authored: Mon Nov 23 14:18:50 2015 -0500
Committer: Mark Payne <ma...@hotmail.com>
Committed: Mon Nov 23 14:18:50 2015 -0500
----------------------------------------------------------------------
.../nifi/provenance/StandardLineageResult.java | 1 +
.../StandardProvenanceEventRecord.java | 1 +
.../asciidoc/images/nifi-connection-menu.png | Bin 0 -> 31189 bytes
.../main/asciidoc/images/nifi-connection.png | Bin 0 -> 166905 bytes
.../asciidoc/images/nifi-process-group-menu.png | Bin 0 -> 98964 bytes
.../asciidoc/images/nifi-processor-menu.png | Bin 0 -> 99962 bytes
.../src/main/asciidoc/images/nifi-rpg-menu.png | Bin 0 -> 108291 bytes
nifi-docs/src/main/asciidoc/user-guide.adoc | 145 +++++--
.../repository/StandardProcessSession.java | 4 +-
.../js/nf/provenance/nf-provenance-table.js | 5 +
.../nifi-hbase-processors/pom.xml | 4 +
.../nifi/hbase/AbstractHBaseProcessor.java | 23 -
.../org/apache/nifi/hbase/AbstractPutHBase.java | 183 ++++++++
.../java/org/apache/nifi/hbase/GetHBase.java | 3 +-
.../org/apache/nifi/hbase/PutHBaseCell.java | 153 +------
.../org/apache/nifi/hbase/PutHBaseJSON.java | 230 ++++++++++
.../org.apache.nifi.processor.Processor | 3 +-
.../org/apache/nifi/hbase/HBaseTestUtil.java | 87 ++++
.../nifi/hbase/MockHBaseClientService.java | 14 +-
.../org/apache/nifi/hbase/TestPutHBaseCell.java | 60 ++-
.../org/apache/nifi/hbase/TestPutHBaseJSON.java | 423 +++++++++++++++++++
.../nifi/provenance/StandardRecordReader.java | 2 +
.../nifi/provenance/StandardRecordWriter.java | 2 +
.../standard/AbstractSyslogProcessor.java | 2 +-
.../nifi/processors/standard/InvokeHTTP.java | 19 +-
.../apache/nifi/hbase/HBaseClientService.java | 11 +
.../org/apache/nifi/hbase/put/PutColumn.java | 47 +++
.../org/apache/nifi/hbase/put/PutFlowFile.java | 38 +-
.../nifi/hbase/HBase_1_1_2_ClientService.java | 25 +-
.../hbase/TestHBase_1_1_2_ClientService.java | 27 +-
30 files changed, 1257 insertions(+), 255 deletions(-)
----------------------------------------------------------------------
[42/50] [abbrv] nifi git commit: NIFI-655: - Refactoring web security
to use Spring Security Java Configuration. - Introducing security in Web UI
in order to get JWT.
Posted by mc...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/login.jsp
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/login.jsp b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/login.jsp
new file mode 100644
index 0000000..978d019
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/login.jsp
@@ -0,0 +1,54 @@
+<%--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+--%>
+<%@ page contentType="text/html" pageEncoding="UTF-8" session="false" %>
+<!DOCTYPE html>
+<html>
+ <head>
+ <title>NiFi Login</title>
+ <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
+ <link rel="shortcut icon" href="images/nifi16.ico"/>
+ <link rel="stylesheet" href="css/reset.css" type="text/css" />
+ ${nf.login.style.tags}
+ <link rel="stylesheet" href="js/jquery/modal/jquery.modal.css?${project.version}" type="text/css" />
+ <link rel="stylesheet" href="js/jquery/qtip2/jquery.qtip.min.css?" type="text/css" />
+ <link rel="stylesheet" href="js/jquery/ui-smoothness/jquery-ui-1.10.4.min.css" type="text/css" />
+ <script type="text/javascript" src="js/jquery/jquery-2.1.1.min.js"></script>
+ <script type="text/javascript" src="js/jquery/jquery.base64.js"></script>
+ <script type="text/javascript" src="js/jquery/jquery.count.js"></script>
+ <script type="text/javascript" src="js/jquery/jquery.center.js"></script>
+ <script type="text/javascript" src="js/jquery/modal/jquery.modal.js?${project.version}"></script>
+ <script type="text/javascript" src="js/jquery/qtip2/jquery.qtip.min.js"></script>
+ <script type="text/javascript" src="js/jquery/ui-smoothness/jquery-ui-1.10.4.min.js"></script>
+ <script type="text/javascript" src="js/nf/nf-namespace.js?${project.version}"></script>
+ ${nf.login.script.tags}
+ </head>
+ <body class="login-body">
+ <div id="user-logout-container" class="hidden">
+ <span id="user-logout" class="link">logout</span>
+ </div>
+ <div id="login-contents-container">
+ <jsp:include page="/WEB-INF/partials/login/login-message.jsp"/>
+ <jsp:include page="/WEB-INF/partials/login/login-form.jsp"/>
+ <jsp:include page="/WEB-INF/partials/login/nifi-registration-form.jsp"/>
+ <jsp:include page="/WEB-INF/partials/login/login-submission.jsp"/>
+ <jsp:include page="/WEB-INF/partials/login/login-progress.jsp"/>
+ </div>
+ <jsp:include page="/WEB-INF/partials/ok-dialog.jsp"/>
+ <div id="faded-background"></div>
+ <div id="glass-pane"></div>
+ </body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/message-page.jsp
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/message-page.jsp b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/message-page.jsp
index 796877f..b0ba026 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/message-page.jsp
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/message-page.jsp
@@ -27,8 +27,8 @@
<body class="message-pane">
<div class="message-pane-message-box">
- <p class="message-pane-title"><%= request.getAttribute("title") == null ? "" : org.apache.nifi.util.EscapeUtils.escapeHtml(request.getAttribute("title").toString()) %></p>
- <p class="message-pane-content"><%= request.getAttribute("messages") == null ? "" : org.apache.nifi.util.EscapeUtils.escapeHtml(request.getAttribute("messages").toString()) %></p>
+ <div class="message-pane-title"><%= request.getAttribute("title") == null ? "" : org.apache.nifi.util.EscapeUtils.escapeHtml(request.getAttribute("title").toString()) %></div>
+ <div class="message-pane-content"><%= request.getAttribute("messages") == null ? "" : org.apache.nifi.util.EscapeUtils.escapeHtml(request.getAttribute("messages").toString()) %></div>
</div>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/provenance.jsp
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/provenance.jsp b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/provenance.jsp
index e02a7cd..496bfd1 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/provenance.jsp
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/provenance.jsp
@@ -31,6 +31,7 @@
<link rel="stylesheet" href="js/jquery/slickgrid/css/slick.grid.css" type="text/css" />
<link rel="stylesheet" href="js/jquery/slickgrid/css/slick-default-theme.css" type="text/css" />
<script type="text/javascript" src="js/jquery/jquery-2.1.1.min.js"></script>
+ <script type="text/javascript" src="js/jquery/jquery.base64.js"></script>
<script type="text/javascript" src="js/jquery/jquery.center.js"></script>
<script type="text/javascript" src="js/jquery/modal/jquery.modal.js?${project.version}"></script>
<script type="text/javascript" src="js/jquery/tabbs/jquery.tabbs.js?${project.version}"></script>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/summary.jsp
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/summary.jsp b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/summary.jsp
index e6f3305..45dff93 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/summary.jsp
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/summary.jsp
@@ -38,6 +38,7 @@
<script type="text/javascript" src="js/codemirror/lib/codemirror-compressed.js"></script>
<script type="text/javascript" src="js/jquery/jquery-2.1.1.min.js"></script>
<script type="text/javascript" src="js/jquery/ui-smoothness/jquery-ui-1.10.4.min.js"></script>
+ <script type="text/javascript" src="js/jquery/jquery.base64.js"></script>
<script type="text/javascript" src="js/jquery/jquery.center.js"></script>
<script type="text/javascript" src="js/jquery/tabbs/jquery.tabbs.js?${project.version}"></script>
<script type="text/javascript" src="js/jquery/combo/jquery.combo.js?${project.version}"></script>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/templates.jsp
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/templates.jsp b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/templates.jsp
index 126c388..452064d 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/templates.jsp
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/templates.jsp
@@ -30,6 +30,7 @@
<link rel="stylesheet" href="js/jquery/slickgrid/css/slick.grid.css" type="text/css" />
<link rel="stylesheet" href="js/jquery/slickgrid/css/slick-default-theme.css" type="text/css" />
<script type="text/javascript" src="js/jquery/jquery-2.1.1.min.js"></script>
+ <script type="text/javascript" src="js/jquery/jquery.base64.js"></script>
<script type="text/javascript" src="js/jquery/jquery.center.js"></script>
<script type="text/javascript" src="js/jquery/jquery.form.min.js"></script>
<script type="text/javascript" src="js/jquery/combo/jquery.combo.js?${project.version}"></script>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/users.jsp
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/users.jsp b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/users.jsp
index a5f422c..b3e0968 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/users.jsp
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/users.jsp
@@ -31,6 +31,7 @@
<link rel="stylesheet" href="js/jquery/slickgrid/css/slick.grid.css" type="text/css" />
<link rel="stylesheet" href="js/jquery/slickgrid/css/slick-default-theme.css" type="text/css" />
<script type="text/javascript" src="js/jquery/jquery-2.1.1.min.js"></script>
+ <script type="text/javascript" src="js/jquery/jquery.base64.js"></script>
<script type="text/javascript" src="js/jquery/jquery.center.js"></script>
<script type="text/javascript" src="js/jquery/tabbs/jquery.tabbs.js?${project.version}"></script>
<script type="text/javascript" src="js/jquery/combo/jquery.combo.js?${project.version}"></script>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/canvas-header.jsp
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/canvas-header.jsp b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/canvas-header.jsp
index f312327..2ea7ca6 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/canvas-header.jsp
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/canvas-header.jsp
@@ -44,6 +44,17 @@
</div>
<div id="header-links-container">
<ul>
+ <li id="current-user-container">
+ <div id="anonymous-user-alert" class="hidden"></div>
+ <div id="current-user"></div>
+ <div class="clear"></div>
+ </li>
+ <li id="login-link-container">
+ <span id="login-link" class="link">login</span>
+ </li>
+ <li id="logout-link-container" style="display: none;">
+ <span id="logout-link" class="link">logout</span>
+ </li>
<li>
<span id="help-link" class="link">help</span>
</li>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/registration.jsp
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/registration.jsp b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/registration.jsp
deleted file mode 100644
index 56b3236..0000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/registration.jsp
+++ /dev/null
@@ -1,44 +0,0 @@
-<%--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
---%>
-<%@ page contentType="text/html" pageEncoding="UTF-8" session="false" %>
-<div id="registration-pane" class="message-pane hidden">
- <div class="message-pane-message-box">
- <p id="register-title" class="message-pane-title">You are not authorized to access this data flow</p>
- <p id="register-content" class="message-pane-content">
- <div>
- <div id="expand-registration-button" class="collapsed pointer"></div>
- <span id="expand-registration-text" class="link">Request Access</span>
- </div>
- <div id="registration-form" class="settings hidden">
- <div class="setting">
- <div class="setting-name">Justification</div>
- <div class="setting-field">
- <textarea cols="30" rows="4" id="registration-justification" maxlength="500" name="registration-justification" class="setting-input"></textarea>
- </div>
- <div style="text-align: right; color: #666; margin-top: 2px;">
- <span id="remaining-characters"></span> characters remaining
- </div>
- <div class="clear"></div>
- </div>
- <div>
- <div id="registration-form-submit" class="button">Submit</div>
- <div class="clear"></div>
- </div>
- </div>
- </p>
- </div>
-</div>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/login/login-form.jsp
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/login/login-form.jsp b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/login/login-form.jsp
new file mode 100644
index 0000000..f8f06f3
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/login/login-form.jsp
@@ -0,0 +1,32 @@
+<%--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+--%>
+<%@ page contentType="text/html" pageEncoding="UTF-8" session="false" %>
+<div id="login-container" class="hidden">
+ <div class="login-title">Log In</div>
+ <div class="setting">
+ <div class="setting-name">Username</div>
+ <div class="setting-field">
+ <input type="text" id="username"/>
+ </div>
+ </div>
+ <div class="setting">
+ <div class="setting-name">Password</div>
+ <div class="setting-field">
+ <input type="password" id="password"/>
+ </div>
+ </div>
+</div>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/login/login-message.jsp
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/login/login-message.jsp b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/login/login-message.jsp
new file mode 100644
index 0000000..053af81
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/login/login-message.jsp
@@ -0,0 +1,21 @@
+<%--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+--%>
+<%@ page contentType="text/html" pageEncoding="UTF-8" session="false" %>
+<div id="login-message-container" class="hidden">
+ <div id="login-message-title"></div>
+ <div id="login-message"></div>
+</div>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/login/login-progress.jsp
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/login/login-progress.jsp b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/login/login-progress.jsp
new file mode 100644
index 0000000..6b08e54
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/login/login-progress.jsp
@@ -0,0 +1,22 @@
+<%--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+--%>
+<%@ page contentType="text/html" pageEncoding="UTF-8" session="false" %>
+<div id="login-progress-container" class="login-container hidden">
+ <div id="login-progress-spinner" class="loading-container"></div>
+ <div id="login-progress-label"></div>
+ <div class="clear"></div>
+</div>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/login/login-submission.jsp
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/login/login-submission.jsp b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/login/login-submission.jsp
new file mode 100644
index 0000000..508ead3
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/login/login-submission.jsp
@@ -0,0 +1,20 @@
+<%--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+--%>
+<%@ page contentType="text/html" pageEncoding="UTF-8" session="false" %>
+<div id="login-submission-container" class="login-container hidden">
+ <div id="login-submission-button" class="button">Log in</div>
+</div>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/login/nifi-registration-form.jsp
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/login/nifi-registration-form.jsp b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/login/nifi-registration-form.jsp
new file mode 100644
index 0000000..3806bd5
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/login/nifi-registration-form.jsp
@@ -0,0 +1,38 @@
+<%--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+--%>
+<%@ page contentType="text/html" pageEncoding="UTF-8" session="false" %>
+<div id="nifi-registration-container" class="login-container hidden">
+ <div id="nifi-registration-title" class="login-title nifi-submit-justification">Submit Justification</div>
+ <div id="nifi-user-submit-justification-container" class="nifi-submit-justification">
+ <div class="setting">
+ <div class="setting-name">User</div>
+ <div class="setting-field">
+ <div id="nifi-user-submit-justification"></div>
+ </div>
+ </div>
+ </div>
+ <div class="setting">
+ <div class="setting-name">Justification</div>
+ <div class="setting-field">
+ <textarea cols="30" rows="4" id="nifi-registration-justification" maxlength="500" class="setting-input"></textarea>
+ </div>
+ <div style="text-align: right; color: #666; margin-top: 2px; float: right;">
+ <span id="remaining-characters"></span> characters remaining
+ </div>
+ <div class="clear"></div>
+ </div>
+</div>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/message-pane.jsp
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/message-pane.jsp b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/message-pane.jsp
index 1bdec3d..db5dece 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/message-pane.jsp
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/message-pane.jsp
@@ -16,8 +16,11 @@
--%>
<%@ page contentType="text/html" pageEncoding="UTF-8" session="false" %>
<div id="message-pane" class="message-pane hidden">
+ <div id="user-logout-container" class="hidden">
+ <span id="user-logout" class="link">logout</span>
+ </div>
<div class="message-pane-message-box">
- <p id="message-title" class="message-pane-title"></p>
- <p id="message-content" class="message-pane-content"></p>
+ <div id="message-title" class="message-pane-title"></div>
+ <div id="message-content" class="message-pane-content"></div>
</div>
</div>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/users/user-details-dialog.jsp
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/users/user-details-dialog.jsp b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/users/user-details-dialog.jsp
index 8587be3..8a81882 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/users/user-details-dialog.jsp
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/users/user-details-dialog.jsp
@@ -25,7 +25,7 @@
<div class="clear"></div>
</div>
<div class="setting">
- <div class="setting-name">DN</div>
+ <div class="setting-name">Identity</div>
<div class="setting-field">
<span id="user-dn-details-dialog"></span>
</div>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/web.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/web.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/web.xml
index d0a5e39..232dbce 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/web.xml
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/web.xml
@@ -17,18 +17,17 @@
<display-name>nifi</display-name>
<!-- servlet to map to canvas page -->
-
<servlet>
<servlet-name>NiFiCanvas</servlet-name>
<jsp-file>/WEB-INF/pages/canvas.jsp</jsp-file>
</servlet>
<servlet-mapping>
<servlet-name>NiFiCanvas</servlet-name>
+ <!--<url-pattern>/token</url-pattern>-->
<url-pattern>/canvas</url-pattern>
</servlet-mapping>
<!-- servlet to map to summary page -->
-
<servlet>
<servlet-name>NiFiSummary</servlet-name>
<jsp-file>/WEB-INF/pages/summary.jsp</jsp-file>
@@ -39,7 +38,6 @@
</servlet-mapping>
<!-- servlet to map to history page -->
-
<servlet>
<servlet-name>NiFiHistory</servlet-name>
<jsp-file>/WEB-INF/pages/history.jsp</jsp-file>
@@ -50,7 +48,6 @@
</servlet-mapping>
<!-- servlet to map to provenance page -->
-
<servlet>
<servlet-name>NiFiProvenance</servlet-name>
<jsp-file>/WEB-INF/pages/provenance.jsp</jsp-file>
@@ -61,7 +58,6 @@
</servlet-mapping>
<!-- servlet to map to counters page -->
-
<servlet>
<servlet-name>NiFiCounters</servlet-name>
<jsp-file>/WEB-INF/pages/counters.jsp</jsp-file>
@@ -72,7 +68,6 @@
</servlet-mapping>
<!-- servlet to map to templates page -->
-
<servlet>
<servlet-name>NiFiTemplates</servlet-name>
<jsp-file>/WEB-INF/pages/templates.jsp</jsp-file>
@@ -83,7 +78,6 @@
</servlet-mapping>
<!-- servlet to map to users page -->
-
<servlet>
<servlet-name>NiFiUsers</servlet-name>
<jsp-file>/WEB-INF/pages/users.jsp</jsp-file>
@@ -94,7 +88,6 @@
</servlet-mapping>
<!-- servlet to map to cluster page -->
-
<servlet>
<servlet-name>NiFiCluster</servlet-name>
<jsp-file>/WEB-INF/pages/cluster.jsp</jsp-file>
@@ -105,7 +98,6 @@
</servlet-mapping>
<!-- servlet to map to bulletin board page -->
-
<servlet>
<servlet-name>BulletinBoard</servlet-name>
<jsp-file>/WEB-INF/pages/bulletin-board.jsp</jsp-file>
@@ -116,7 +108,6 @@
</servlet-mapping>
<!-- servlet to support message page -->
-
<servlet>
<servlet-name>MessagePage</servlet-name>
<jsp-file>/WEB-INF/pages/message-page.jsp</jsp-file>
@@ -127,7 +118,6 @@
</servlet-mapping>
<!-- servlet to support image downloading -->
-
<servlet>
<servlet-name>DownloadSvg</servlet-name>
<servlet-class>org.apache.nifi.web.servlet.DownloadSvg</servlet-class>
@@ -137,6 +127,16 @@
<url-pattern>/download-svg</url-pattern>
</servlet-mapping>
+ <!-- servlet to login page -->
+ <servlet>
+ <servlet-name>Login</servlet-name>
+ <jsp-file>/WEB-INF/pages/login.jsp</jsp-file>
+ </servlet>
+ <servlet-mapping>
+ <servlet-name>Login</servlet-name>
+ <url-pattern>/login</url-pattern>
+ </servlet-mapping>
+
<filter>
<filter-name>IeEdgeHeader</filter-name>
<filter-class>org.apache.nifi.web.filter.IeEdgeHeader</filter-class>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/canvas.css
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/canvas.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/canvas.css
index a8866ad..abb5ebd 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/canvas.css
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/canvas.css
@@ -28,7 +28,6 @@
@import url(connection-configuration.css);
@import url(connection-details.css);
@import url(shell.css);
-@import url(registration.css);
@import url(dialog.css);
@import url(new-processor-dialog.css);
@import url(new-controller-service-dialog.css);
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/header.css
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/header.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/header.css
index 0f08b47..49dd3a0 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/header.css
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/header.css
@@ -506,6 +506,24 @@ div.search-glass-pane {
/* styles for the status link */
+#anonymous-user-alert {
+ float: left;
+ margin-top: -2px;
+ margin-right: 6px;
+ width: 18px;
+ height: 16px;
+ background-image: url(../images/iconAlert.png);
+}
+
+#current-user {
+ float: left;
+ margin-right: 8px;
+ font-weight: bold;
+ max-width: 250px;
+ text-overflow: ellipsis;
+ overflow: hidden;
+}
+
#utilities-container {
float: right;
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/login.css
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/login.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/login.css
new file mode 100644
index 0000000..68086a7
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/login.css
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ Login Styles
+*/
+
+#login-contents-container {
+ position: absolute;
+ top: 0px;
+ left: 0px;
+ bottom: 0px;
+ right: 0px;
+ background: #fff url(../images/bg-error.png) left top no-repeat;
+ font-family: Verdana, Geneva, sans-serif;
+ padding-top: 100px;
+ padding-left: 100px;
+}
+
+#login-message-title {
+ font-size: 18px;
+ color: #294c58;
+ margin-bottom: 16px;
+}
+
+#login-message {
+ font-size: 11px;
+}
+
+.login-title {
+ font-size: 12px;
+ font-weight: bold;
+ margin-bottom: 10px;
+ color: #000;
+}
+
+/*
+ Login
+*/
+
+body.login-body input, body.login-body textarea {
+ width: 400px;
+}
+
+div.login-container {
+ width: 412px;
+}
+
+/*
+ NiFi Registration
+*/
+
+#nifi-user-submit-justification-container {
+ margin-bottom: 10px;
+}
+
+#nifi-user-submit-justification {
+ font-weight: bold;
+}
+
+#nifi-registration-justification {
+ height: 200px;
+}
+
+/*
+ Login Progress
+*/
+
+#login-progress-label {
+ float: right;
+ font-weight: bold;
+ line-height: 16px;
+}
+
+#login-progress-spinner {
+ float: right;
+ width: 16px;
+ height: 16px;
+ background-color: transparent;
+ margin-left: 3px;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/main.css
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/main.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/main.css
index 95ee641..deadcd5 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/main.css
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/main.css
@@ -77,6 +77,17 @@ div.context-menu-provenance {
background-position: top left;
}
+#user-logout-container {
+ position: absolute;
+ left: 478px;
+ top: 100px;
+ z-index: 1300;
+}
+
+#user-logout {
+ text-decoration: underline;
+}
+
/*
General Styles
*/
@@ -153,7 +164,7 @@ input.filter-list {
background: transparent url(../images/iconTwistArrow.png) no-repeat scroll top right;
}
-input[type=text], textarea {
+input[type=text], input[type=password], textarea {
background: white url(../images/bgInputText.png) repeat-x scroll top;
border: 1px solid #ccc;
font-family: Verdana;
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/registration.css
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/registration.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/registration.css
deleted file mode 100644
index d4fdc7e..0000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/registration.css
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/*
- Registration form styles.
-*/
-
-#registration-pane {
- z-index: 1299;
-}
-
-#registration-form {
- margin-top: 10px;
- width: 610px;
-}
-
-#expand-registration-button {
- width: 10px;
- height: 10px;
- float: left;
- margin-right: 5px;
-}
-
-#expand-registration-text {
- -webkit-user-select: none;
- -moz-user-select: none;
-}
-
-#registration-justification {
- width: 600px;
- height: 200px;
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/jquery.base64.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/jquery.base64.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/jquery.base64.js
new file mode 100644
index 0000000..32ceab0
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/jquery.base64.js
@@ -0,0 +1,123 @@
+/*!
+ * jquery.base64.js 0.0.3 - https://github.com/yckart/jquery.base64.js
+ * Makes Base64 en & -decoding simpler as it is.
+ *
+ * Based upon: https://gist.github.com/Yaffle/1284012
+ *
+ * Copyright (c) 2012 Yannick Albert (http://yckart.com)
+ * Licensed under the MIT license (http://www.opensource.org/licenses/mit-license.php).
+ * 2013/02/10
+ **/
+;(function($) {
+
+ var b64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/",
+ a256 = '',
+ r64 = [256],
+ r256 = [256],
+ i = 0;
+
+ var UTF8 = {
+
+ /**
+ * Encode multi-byte Unicode string into utf-8 multiple single-byte characters
+ * (BMP / basic multilingual plane only)
+ *
+ * Chars in range U+0080 - U+07FF are encoded in 2 chars, U+0800 - U+FFFF in 3 chars
+ *
+ * @param {String} strUni Unicode string to be encoded as UTF-8
+ * @returns {String} encoded string
+ */
+ encode: function(strUni) {
+ // use regular expressions & String.replace callback function for better efficiency
+ // than procedural approaches
+ var strUtf = strUni.replace(/[\u0080-\u07ff]/g, // U+0080 - U+07FF => 2 bytes 110yyyyy, 10zzzzzz
+ function(c) {
+ var cc = c.charCodeAt(0);
+ return String.fromCharCode(0xc0 | cc >> 6, 0x80 | cc & 0x3f);
+ })
+ .replace(/[\u0800-\uffff]/g, // U+0800 - U+FFFF => 3 bytes 1110xxxx, 10yyyyyy, 10zzzzzz
+ function(c) {
+ var cc = c.charCodeAt(0);
+ return String.fromCharCode(0xe0 | cc >> 12, 0x80 | cc >> 6 & 0x3F, 0x80 | cc & 0x3f);
+ });
+ return strUtf;
+ },
+
+ /**
+ * Decode utf-8 encoded string back into multi-byte Unicode characters
+ *
+ * @param {String} strUtf UTF-8 string to be decoded back to Unicode
+ * @returns {String} decoded string
+ */
+ decode: function(strUtf) {
+ // note: decode 3-byte chars first as decoded 2-byte strings could appear to be 3-byte char!
+ var strUni = strUtf.replace(/[\u00e0-\u00ef][\u0080-\u00bf][\u0080-\u00bf]/g, // 3-byte chars
+ function(c) { // (note parentheses for precence)
+ var cc = ((c.charCodeAt(0) & 0x0f) << 12) | ((c.charCodeAt(1) & 0x3f) << 6) | (c.charCodeAt(2) & 0x3f);
+ return String.fromCharCode(cc);
+ })
+ .replace(/[\u00c0-\u00df][\u0080-\u00bf]/g, // 2-byte chars
+ function(c) { // (note parentheses for precence)
+ var cc = (c.charCodeAt(0) & 0x1f) << 6 | c.charCodeAt(1) & 0x3f;
+ return String.fromCharCode(cc);
+ });
+ return strUni;
+ }
+ };
+
+ while(i < 256) {
+ var c = String.fromCharCode(i);
+ a256 += c;
+ r256[i] = i;
+ r64[i] = b64.indexOf(c);
+ ++i;
+ }
+
+ function code(s, discard, alpha, beta, w1, w2) {
+ s = String(s);
+ var buffer = 0,
+ i = 0,
+ length = s.length,
+ result = '',
+ bitsInBuffer = 0;
+
+ while(i < length) {
+ var c = s.charCodeAt(i);
+ c = c < 256 ? alpha[c] : -1;
+
+ buffer = (buffer << w1) + c;
+ bitsInBuffer += w1;
+
+ while(bitsInBuffer >= w2) {
+ bitsInBuffer -= w2;
+ var tmp = buffer >> bitsInBuffer;
+ result += beta.charAt(tmp);
+ buffer ^= tmp << bitsInBuffer;
+ }
+ ++i;
+ }
+ if(!discard && bitsInBuffer > 0) result += beta.charAt(buffer << (w2 - bitsInBuffer));
+ return result;
+ }
+
+ var Plugin = $.base64 = function(dir, input, encode) {
+ return input ? Plugin[dir](input, encode) : dir ? null : this;
+ };
+
+ Plugin.btoa = Plugin.encode = function(plain, utf8encode) {
+ plain = Plugin.raw === false || Plugin.utf8encode || utf8encode ? UTF8.encode(plain) : plain;
+ plain = code(plain, false, r256, b64, 8, 6);
+ return plain + '===='.slice((plain.length % 4) || 4);
+ };
+
+ Plugin.atob = Plugin.decode = function(coded, utf8decode) {
+ coded = coded.replace(/[^A-Za-z0-9\+\/\=]/g, "");
+ coded = String(coded).split('=');
+ var i = coded.length;
+ do {--i;
+ coded[i] = code(coded[i], true, r64, a256, 6, 8);
+ } while (i > 0);
+ coded = coded.join('');
+ return Plugin.raw === false || Plugin.utf8decode || utf8decode ? UTF8.decode(coded) : coded;
+ };
+}(jQuery));
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/bulletin-board/nf-bulletin-board.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/bulletin-board/nf-bulletin-board.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/bulletin-board/nf-bulletin-board.js
index 656755e..55d0a9d 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/bulletin-board/nf-bulletin-board.js
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/bulletin-board/nf-bulletin-board.js
@@ -417,6 +417,8 @@ nf.BulletinBoard = (function () {
* Initializes the bulletin board page.
*/
init: function () {
+ nf.Storage.init();
+
initializePage().done(function () {
start();
});
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-canvas-header.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-canvas-header.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-canvas-header.js
index 5cc1eff..7d63534 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-canvas-header.js
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-canvas-header.js
@@ -31,8 +31,10 @@ nf.CanvasHeader = (function () {
return {
/**
* Initialize the canvas header.
+ *
+ * @argument {boolean} supportsLogin Whether login is supported
*/
- init: function () {
+ init: function (supportsLogin) {
// mouse over for the reporting link
nf.Common.addHoverEffect('#reporting-link', 'reporting-link', 'reporting-link-hover').click(function () {
nf.Shell.showPage('summary');
@@ -139,6 +141,28 @@ nf.CanvasHeader = (function () {
nf.Shell.showPage(config.urls.helpDocument);
});
+ // show the login link if supported and user is currently anonymous
+ var isAnonymous = $('#current-user').text() === nf.Common.ANONYMOUS_USER_TEXT;
+ if (supportsLogin === true && isAnonymous) {
+ // login link
+ $('#login-link').click(function () {
+ nf.Shell.showPage('login', false);
+ });
+ } else {
+ $('#login-link-container').css('display', 'none');
+ }
+
+ // if login is not supported, don't show the current user
+ if (supportsLogin === false) {
+ $('#current-user-container').css('display', 'none');
+ }
+
+ // logout link
+ $('#logout-link').click(function () {
+ nf.Storage.removeItem("jwt");
+ window.location = '/nifi';
+ });
+
// initialize the new template dialog
$('#new-template-dialog').modal({
headerText: 'Create Template',
@@ -154,11 +178,11 @@ nf.CanvasHeader = (function () {
handler: {
click: function () {
var selection = nf.CanvasUtils.getSelection();
-
+
// color the selected components
selection.each(function (d) {
var selected = d3.select(this);
-
+
var revision = nf.Client.getRevision();
var selectedData = selected.datum();
@@ -197,7 +221,7 @@ nf.CanvasHeader = (function () {
});
}
});
-
+
// close the dialog
$('#fill-color-dialog').modal('hide');
}
@@ -238,20 +262,20 @@ nf.CanvasHeader = (function () {
});
}
});
-
+
// updates the color if its a valid hex color string
var updateColor = function () {
var hex = $('#fill-color-value').val();
-
+
// only update the fill color when its a valid hex color string
// #[six hex characters|three hex characters] case insensitive
if (/(^#[0-9A-F]{6}$)|(^#[0-9A-F]{3}$)/i.test(hex)) {
$('#fill-color').minicolors('value', hex);
}
};
-
+
// apply fill color from field on blur and enter press
- $('#fill-color-value').on('blur', updateColor).on('keyup', function(e) {
+ $('#fill-color-value').on('blur', updateColor).on('keyup', function (e) {
var code = e.keyCode ? e.keyCode : e.which;
if (code === $.ui.keyCode.ENTER) {
updateColor();
@@ -310,23 +334,22 @@ nf.CanvasHeader = (function () {
}
}
});
-
+
var toolbar = $('#toolbar');
var groupButton = $('#action-group');
- $(window).on('resize', function() {
+ $(window).on('resize', function () {
if (toolbar.width() < MIN_TOOLBAR_WIDTH && groupButton.is(':visible')) {
toolbar.find('.secondary').hide();
} else if (toolbar.width() > MIN_TOOLBAR_WIDTH && groupButton.is(':hidden')) {
toolbar.find('.secondary').show();
}
});
-
+
// set up the initial visibility
if (toolbar.width() < MIN_TOOLBAR_WIDTH) {
toolbar.find('.secondary').hide();
}
},
-
/**
* Reloads and clears any warnings.
*/
@@ -341,7 +364,7 @@ nf.CanvasHeader = (function () {
// hide the refresh link on the canvas
$('#stats-last-refreshed').removeClass('alert');
$('#refresh-required-container').hide();
-
+
// hide the refresh link on the settings
$('#settings-last-refreshed').removeClass('alert');
$('#settings-refresh-required-icon').hide();
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-canvas.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-canvas.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-canvas.js
index 7137fe4..c9498fe 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-canvas.js
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-canvas.js
@@ -57,6 +57,7 @@ nf.Canvas = (function () {
var config = {
urls: {
+ identity: '../nifi-api/controller/identity',
authorities: '../nifi-api/controller/authorities',
revision: '../nifi-api/controller/revision',
status: '../nifi-api/controller/status',
@@ -64,6 +65,7 @@ nf.Canvas = (function () {
banners: '../nifi-api/controller/banners',
controller: '../nifi-api/controller',
controllerConfig: '../nifi-api/controller/config',
+ accessConfig: '../nifi-api/access/config',
cluster: '../nifi-api/cluster',
d3Script: 'js/d3/d3.min.js'
}
@@ -1018,122 +1020,166 @@ nf.Canvas = (function () {
* Initialize NiFi.
*/
init: function () {
- // init the registration form before performing the first query since
- // the response could lead to a registration attempt
- nf.Registration.init();
+ // get the current user's identity
+ var identityXhr = $.ajax({
+ type: 'GET',
+ url: config.urls.identity,
+ dataType: 'json'
+ });
- // get the controller config to register the status poller
- var configXhr = $.ajax({
+ // get the current user's authorities
+ var authoritiesXhr = $.ajax({
type: 'GET',
- url: config.urls.controllerConfig,
+ url: config.urls.authorities,
dataType: 'json'
});
- // create the deferred cluster request
- var isClusteredRequest = $.Deferred(function (deferred) {
- $.ajax({
- type: 'HEAD',
- url: config.urls.cluster
- }).done(function (response, status, xhr) {
- clustered = true;
- deferred.resolve(response, status, xhr);
+ // load the identity and authorities for the current user
+ var userXhr = $.Deferred(function (deferred) {
+ $.when(authoritiesXhr, identityXhr).done(function (authoritiesResult, identityResult) {
+ var authoritiesResponse = authoritiesResult[0];
+ var identityResponse = identityResult[0];
+
+ // set the user's authorities
+ nf.Common.setAuthorities(authoritiesResponse.authorities);
+
+ // at this point the user may be themselves or anonymous
+
+ // if the user is logged, we want to determine if they were logged in using a certificate
+ if (identityResponse.identity !== 'anonymous') {
+ // rendner the users name
+ $('#current-user').text(identityResponse.identity).show();
+
+ // render the logout button if there is a token locally
+ if (nf.Storage.getItem('jwt') !== null) {
+ $('#logout-link-container').show();
+ }
+ } else {
+ // set the anonymous user label
+ nf.Common.setAnonymousUserLabel();
+ }
+ deferred.resolve();
}).fail(function (xhr, status, error) {
- if (xhr.status === 404) {
- clustered = false;
- deferred.resolve('', 'success', xhr);
+ // there is no anonymous access and we don't know this user - open the login page which handles login/registration/etc
+ if (xhr.status === 401 || xhr.status === 403) {
+ window.location = '/nifi/login';
} else {
deferred.reject(xhr, status, error);
}
});
}).promise();
- // load the authorities
- var authoritiesXhr = $.ajax({
- type: 'GET',
- url: config.urls.authorities,
- dataType: 'json'
- });
+ userXhr.done(function () {
+ // get the controller config to register the status poller
+ var configXhr = $.ajax({
+ type: 'GET',
+ url: config.urls.controllerConfig,
+ dataType: 'json'
+ });
- // ensure the authorities and config request is processed first
- $.when(authoritiesXhr, configXhr).done(function (authoritiesResult, configResult) {
- var authoritiesResponse = authoritiesResult[0];
- var configResponse = configResult[0];
+ // get the login config
+ var loginXhr = $.ajax({
+ type: 'GET',
+ url: config.urls.accessConfig,
+ dataType: 'json'
+ });
- // set the user's authorities
- nf.Common.setAuthorities(authoritiesResponse.authorities);
+ // create the deferred cluster request
+ var isClusteredRequest = $.Deferred(function (deferred) {
+ $.ajax({
+ type: 'HEAD',
+ url: config.urls.cluster
+ }).done(function (response, status, xhr) {
+ clustered = true;
+ deferred.resolve(response, status, xhr);
+ }).fail(function (xhr, status, error) {
+ if (xhr.status === 404) {
+ clustered = false;
+ deferred.resolve('', 'success', xhr);
+ } else {
+ deferred.reject(xhr, status, error);
+ }
+ });
+ }).promise();
- // calculate the canvas offset
- var canvasContainer = $('#canvas-container');
- nf.Canvas.CANVAS_OFFSET = canvasContainer.offset().top;
-
- // get the config details
- var configDetails = configResponse.config;
-
- // when both request complete, load the application
- isClusteredRequest.done(function () {
- // get the auto refresh interval
- var autoRefreshIntervalSeconds = parseInt(configDetails.autoRefreshIntervalSeconds, 10);
-
- // initialize whether site to site is secure
- secureSiteToSite = configDetails.siteToSiteSecure;
-
- // load d3
- loadD3().done(function () {
- nf.Storage.init();
-
- // initialize the application
- initCanvas();
- nf.Canvas.View.init();
- nf.ContextMenu.init();
- nf.CanvasToolbar.init();
- nf.CanvasToolbox.init();
- nf.CanvasHeader.init();
- nf.GraphControl.init();
- nf.Search.init();
- nf.Settings.init();
- nf.Actions.init();
-
- // initialize the component behaviors
- nf.Draggable.init();
- nf.Selectable.init();
- nf.Connectable.init();
-
- // initialize the chart
- nf.StatusHistory.init(configDetails.timeOffset);
-
- // initialize the birdseye
- nf.Birdseye.init();
-
- // initialize components
- nf.ConnectionConfiguration.init();
- nf.ControllerService.init();
- nf.ReportingTask.init();
- nf.ProcessorConfiguration.init();
- nf.ProcessGroupConfiguration.init();
- nf.RemoteProcessGroupConfiguration.init();
- nf.RemoteProcessGroupPorts.init();
- nf.PortConfiguration.init();
- nf.SecurePortConfiguration.init();
- nf.LabelConfiguration.init();
- nf.ProcessorDetails.init();
- nf.ProcessGroupDetails.init();
- nf.PortDetails.init();
- nf.SecurePortDetails.init();
- nf.ConnectionDetails.init();
- nf.RemoteProcessGroupDetails.init();
- nf.GoTo.init();
- nf.Graph.init().done(function () {
- // determine the split between the polling
- var pollingSplit = autoRefreshIntervalSeconds / 2;
-
- // register the revision and status polling
- startRevisionPolling(autoRefreshIntervalSeconds);
- setTimeout(function () {
- startStatusPolling(autoRefreshIntervalSeconds);
- }, pollingSplit * 1000);
-
- // hide the splash screen
- nf.Canvas.hideSplash();
+ // ensure the config requests are loaded
+ $.when(configXhr, loginXhr, userXhr).done(function (configResult, loginResult) {
+ var configResponse = configResult[0];
+ var loginResponse = loginResult[0];
+
+ // calculate the canvas offset
+ var canvasContainer = $('#canvas-container');
+ nf.Canvas.CANVAS_OFFSET = canvasContainer.offset().top;
+
+ // get the config details
+ var configDetails = configResponse.config;
+ var loginDetails = loginResponse.config;
+
+ // when both request complete, load the application
+ isClusteredRequest.done(function () {
+ // get the auto refresh interval
+ var autoRefreshIntervalSeconds = parseInt(configDetails.autoRefreshIntervalSeconds, 10);
+
+ // initialize whether site to site is secure
+ secureSiteToSite = configDetails.siteToSiteSecure;
+
+ // load d3
+ loadD3().done(function () {
+ nf.Storage.init();
+
+ // initialize the application
+ initCanvas();
+ nf.Canvas.View.init();
+ nf.ContextMenu.init();
+ nf.CanvasToolbar.init();
+ nf.CanvasToolbox.init();
+ nf.CanvasHeader.init(loginDetails.supportsLogin);
+ nf.GraphControl.init();
+ nf.Search.init();
+ nf.Settings.init();
+
+ // initialize the component behaviors
+ nf.Draggable.init();
+ nf.Selectable.init();
+ nf.Connectable.init();
+
+ // initialize the chart
+ nf.StatusHistory.init(configDetails.timeOffset);
+
+ // initialize the birdseye
+ nf.Birdseye.init();
+
+ // initialize components
+ nf.ConnectionConfiguration.init();
+ nf.ControllerService.init();
+ nf.ReportingTask.init();
+ nf.ProcessorConfiguration.init();
+ nf.ProcessGroupConfiguration.init();
+ nf.RemoteProcessGroupConfiguration.init();
+ nf.RemoteProcessGroupPorts.init();
+ nf.PortConfiguration.init();
+ nf.SecurePortConfiguration.init();
+ nf.LabelConfiguration.init();
+ nf.ProcessorDetails.init();
+ nf.ProcessGroupDetails.init();
+ nf.PortDetails.init();
+ nf.SecurePortDetails.init();
+ nf.ConnectionDetails.init();
+ nf.RemoteProcessGroupDetails.init();
+ nf.GoTo.init();
+ nf.Graph.init().done(function () {
+ // determine the split between the polling
+ var pollingSplit = autoRefreshIntervalSeconds / 2;
+
+ // register the revision and status polling
+ startRevisionPolling(autoRefreshIntervalSeconds);
+ setTimeout(function () {
+ startStatusPolling(autoRefreshIntervalSeconds);
+ }, pollingSplit * 1000);
+
+ // hide the splash screen
+ nf.Canvas.hideSplash();
+ }).fail(nf.Common.handleAjaxError);
}).fail(nf.Common.handleAjaxError);
}).fail(nf.Common.handleAjaxError);
}).fail(nf.Common.handleAjaxError);
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-registration.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-registration.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-registration.js
deleted file mode 100644
index b678e27..0000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-registration.js
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/* global nf */
-
-nf.Registration = (function () {
-
- var config = {
- urls: {
- users: '../nifi-api/controller/users'
- }
- };
-
- return {
- /**
- * Initializes the user account registration form.
- */
- init: function () {
- $('#registration-justification').count({
- charCountField: '#remaining-characters'
- });
-
- // register a click listener to expand/collapse the registration form
- $('#expand-registration-button, #expand-registration-text').click(function () {
- var registrationForm = $('#registration-form');
- if (registrationForm.is(':visible')) {
- $('#expand-registration-button').removeClass('registration-expanded').addClass('collapsed');
- } else {
- $('#expand-registration-button').removeClass('registration-collapsed').addClass('expanded');
- }
- registrationForm.toggle();
- });
-
- // register a click listener for submitting user account requests
- $('#registration-form-submit').one('click', function () {
- var justification = $('#registration-justification').val();
-
- // attempt to create the user account registration
- $.ajax({
- type: 'POST',
- url: config.urls.users,
- data: {
- 'justification': justification
- }
- }).done(function (response) {
- // hide the registration pane
- $('#registration-pane').hide();
-
- // show the message pane
- $('#message-pane').show();
- $('#message-title').text('Thanks');
- $('#message-content').text('Your request will be processed shortly.');
- }).fail(nf.Common.handleAjaxError);
- });
- }
- };
-}());
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-storage.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-storage.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-storage.js
deleted file mode 100644
index 3937970..0000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-storage.js
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/* global nf, d3 */
-
-nf.Storage = (function () {
-
- // Store items for two days before being eligible for removal.
- var TWO_DAYS = 86400000 * 2;
-
- // alternative to local storage thats used when localStorage isn't available
- var alternativeStorage;
-
- // determine if this browser supports local storage
- var SUPPORTS_LOCAL_STORAGE = (function () {
- var test = 'test';
- try {
- localStorage.setItem(test, test);
- localStorage.removeItem(test);
- return true;
- } catch (e) {
- return false;
- }
- }());
-
- return {
- /**
- * Initializes the storage. When the browser supports localStorage,
- * the items will be persisted for two days. Once the scripts runs
- * thereafter, all eligible items will be removed. If localStorage
- * is not supported, a data structure will be used to hold items.
- * This strategy does not support persistence.
- */
- init: function () {
- // if we support local storage, see if anything can be removed
- if (SUPPORTS_LOCAL_STORAGE) {
- for (var i = 0; i < localStorage.length; i++) {
- try {
- // get the next item
- var key = localStorage.key(i);
- var entry = JSON.parse(localStorage.getItem(key));
-
- // get the expiration
- var expires = new Date(entry.expires);
- var now = new Date();
-
- // if the expiration date has passed, remove it
- if (expires.valueOf() < now.valueOf()) {
- localStorage.removeItem(key);
- }
- } catch (e) {
- // likely unable to parse the item
- }
- }
- } else {
- alternativeStorage = d3.map();
- }
- },
-
- /**
- * Stores the specified item. If supported, will be persisted
- * in localStorage.
- *
- * @param {type} key
- * @param {type} item
- */
- setItem: function (key, item) {
- if (SUPPORTS_LOCAL_STORAGE) {
- // calculate the expiration
- var expires = new Date().valueOf() + TWO_DAYS;
-
- // create the entry
- var entry = {
- expires: expires,
- item: item
- };
-
- // store the item
- localStorage.setItem(key, JSON.stringify(entry));
- } else {
- alternativeStorage.set(key, item);
- }
- },
-
- /**
- * Gets the item with the specified key. If an item with this key does
- * not exist, null is returned. If an item exists but cannot be parsed
- * or is malformed/unrecognized, null is returned.
- *
- * @param {type} key
- */
- getItem: function (key) {
- if (SUPPORTS_LOCAL_STORAGE) {
- try {
- // parse the entry
- var entry = JSON.parse(localStorage.getItem(key));
-
- // ensure the entry and item are present
- if (nf.Common.isDefinedAndNotNull(entry) && nf.Common.isDefinedAndNotNull(entry.item)) {
- return entry.item;
- } else {
- return null;
- }
- } catch (e) {
- return null;
- }
- } else {
- return alternativeStorage.get(key);
- }
- },
-
- /**
- * Removes the item with the specified key.
- *
- * @param {type} key
- */
- removeItem: function (key) {
- if (SUPPORTS_LOCAL_STORAGE) {
- localStorage.removeItem(key);
- } else {
- alternativeStorage.remove(key);
- }
- }
- };
-}());
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/cluster/nf-cluster.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/cluster/nf-cluster.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/cluster/nf-cluster.js
index 5f00e2c..1eafa67 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/cluster/nf-cluster.js
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/cluster/nf-cluster.js
@@ -123,6 +123,8 @@ nf.Cluster = (function () {
* Initializes the counters page.
*/
init: function () {
+ nf.Storage.init();
+
// load the users authorities
loadAuthorities().done(function () {
// create the counters table
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/counters/nf-counters.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/counters/nf-counters.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/counters/nf-counters.js
index cbfe336..6af3d20 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/counters/nf-counters.js
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/counters/nf-counters.js
@@ -123,6 +123,8 @@ nf.Counters = (function () {
* Initializes the counters page.
*/
init: function () {
+ nf.Storage.init();
+
// load the users authorities
loadAuthorities().done(function () {
// create the counters table
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/history/nf-history.js
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/history/nf-history.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/history/nf-history.js
index 516507c..8f749c0 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/history/nf-history.js
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/history/nf-history.js
@@ -124,6 +124,8 @@ nf.History = (function () {
* Initializes the status page.
*/
init: function () {
+ nf.Storage.init();
+
// load the users authorities
loadAuthorities().done(function () {
// create the history table
[28/50] [abbrv] nifi git commit: NIFI-1086 Provide refactoring of
InvokeHTTP NIFI-980 Add support for HTTP Digest authentication to InvokeHttp
NIFI-1080 Provide additional InvokeHttp unit tests NIFI-1133 InvokeHTTP
Processor does not save Location header
Posted by mc...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/8c2323dc/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestInvokeHttpCommon.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestInvokeHttpCommon.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestInvokeHttpCommon.java
index 88dfcdb..b44f015 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestInvokeHttpCommon.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestInvokeHttpCommon.java
@@ -24,9 +24,16 @@ import org.apache.nifi.provenance.ProvenanceEventRecord;
import org.apache.nifi.provenance.ProvenanceEventType;
import org.apache.nifi.util.MockFlowFile;
import org.apache.nifi.util.TestRunner;
+import org.eclipse.jetty.security.ConstraintSecurityHandler;
+import org.eclipse.jetty.security.DefaultIdentityService;
+import org.eclipse.jetty.security.HashLoginService;
+import org.eclipse.jetty.security.ServerAuthException;
+import org.eclipse.jetty.security.authentication.DigestAuthenticator;
+import org.eclipse.jetty.server.Authentication;
import org.eclipse.jetty.server.Handler;
import org.eclipse.jetty.server.Request;
import org.eclipse.jetty.server.handler.AbstractHandler;
+import org.eclipse.jetty.util.security.Password;
import org.junit.Assert;
import org.junit.Test;
@@ -67,7 +74,7 @@ public abstract class TestInvokeHttpCommon {
final DateHandler dh = new DateHandler();
addHandler(dh);
- runner.setProperty(InvokeHTTP.Config.PROP_URL, url);
+ runner.setProperty(InvokeHTTP.PROP_URL, url);
createFlowFiles(runner);
runner.run();
@@ -94,47 +101,368 @@ public abstract class TestInvokeHttpCommon {
public void test200() throws Exception {
addHandler(new GetOrHeadHandler());
- runner.setProperty(InvokeHTTP.Config.PROP_URL, url + "/status/200");
+ runner.setProperty(InvokeHTTP.PROP_URL, url + "/status/200");
createFlowFiles(runner);
+ // Verify only one FlowFile gets created/sent
+ runner.run();
runner.run();
- runner.assertTransferCount(InvokeHTTP.Config.REL_SUCCESS_REQ, 1);
- runner.assertTransferCount(InvokeHTTP.Config.REL_SUCCESS_RESP, 1);
- runner.assertTransferCount(InvokeHTTP.Config.REL_RETRY, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_NO_RETRY, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_FAILURE, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
// expected in request status.code and status.message
// original flow file (+attributes)
- final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.Config.REL_SUCCESS_REQ).get(0);
+ final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.REL_SUCCESS_REQ).get(0);
bundle.assertContentEquals("Hello".getBytes("UTF-8"));
- bundle.assertAttributeEquals(InvokeHTTP.Config.STATUS_CODE, "200");
- bundle.assertAttributeEquals(InvokeHTTP.Config.STATUS_MESSAGE, "OK");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
bundle.assertAttributeEquals("Foo", "Bar");
// expected in response
// status code, status message, all headers from server response --> ff attributes
// server response message body into payload of ff
- final MockFlowFile bundle1 = runner.getFlowFilesForRelationship(InvokeHTTP.Config.REL_SUCCESS_RESP).get(0);
+ final MockFlowFile bundle1 = runner.getFlowFilesForRelationship(InvokeHTTP.REL_RESPONSE).get(0);
bundle1.assertContentEquals("/status/200".getBytes("UTF-8"));
- bundle1.assertAttributeEquals(InvokeHTTP.Config.STATUS_CODE, "200");
- bundle1.assertAttributeEquals(InvokeHTTP.Config.STATUS_MESSAGE, "OK");
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
+ bundle1.assertAttributeEquals("Foo", "Bar");
+ bundle1.assertAttributeEquals("Content-Type", "text/plain; charset=ISO-8859-1");
+
+ final List<ProvenanceEventRecord> provEvents = runner.getProvenanceEvents();
+ assertEquals(2, provEvents.size());
+ boolean forkEvent = false;
+ boolean fetchEvent = false;
+ for (final ProvenanceEventRecord event : provEvents) {
+ if (event.getEventType() == ProvenanceEventType.FORK) {
+ forkEvent = true;
+ } else if (event.getEventType() == ProvenanceEventType.FETCH) {
+ fetchEvent = true;
+ }
+ }
+
+ assertTrue(forkEvent);
+ assertTrue(fetchEvent);
+ }
+
+ @Test
+ public void testOutputResponseRegardless() throws Exception {
+ addHandler(new GetOrHeadHandler());
+
+ runner.setProperty(InvokeHTTP.PROP_URL, url + "/status/200");
+ runner.setProperty(InvokeHTTP.PROP_METHOD, "POST");
+ runner.setProperty(InvokeHTTP.PROP_OUTPUT_RESPONSE_REGARDLESS,"true");
+
+ createFlowFiles(runner);
+
+ runner.run();
+
+ runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
+
+ // expected in request status.code and status.message
+ // original flow file (+attributes)
+ final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.REL_NO_RETRY).get(0);
+ bundle.assertContentEquals("Hello".getBytes("UTF-8"));
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "404");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "Not Found");
+ bundle.assertAttributeEquals("Foo", "Bar");
+
+ // expected in response
+ // status code, status message, all headers from server response --> ff attributes
+ // server response message body into payload of ff
+ final MockFlowFile bundle1 = runner.getFlowFilesForRelationship(InvokeHTTP.REL_RESPONSE).get(0);
+ bundle1.assertContentEquals("NO".getBytes("UTF-8"));
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "404");
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "Not Found");
bundle1.assertAttributeEquals("Foo", "Bar");
bundle1.assertAttributeEquals("Content-Type", "text/plain; charset=ISO-8859-1");
}
@Test
+ public void testOutputResponseRegardlessWithOutputInAttribute() throws Exception {
+ addHandler(new GetOrHeadHandler());
+
+ runner.setProperty(InvokeHTTP.PROP_URL, url + "/status/200");
+ runner.setProperty(InvokeHTTP.PROP_METHOD, "POST");
+ runner.setProperty(InvokeHTTP.PROP_OUTPUT_RESPONSE_REGARDLESS,"true");
+ runner.setProperty(InvokeHTTP.PROP_PUT_OUTPUT_IN_ATTRIBUTE,"outputBody");
+
+ createFlowFiles(runner);
+
+ runner.run();
+
+ runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
+
+ // expected in request status.code and status.message
+ // original flow file (+attributes)
+ final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.REL_NO_RETRY).get(0);
+ bundle.assertContentEquals("Hello".getBytes("UTF-8"));
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "404");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "Not Found");
+ bundle.assertAttributeEquals("outputBody", "NO");
+ bundle.assertAttributeEquals("Foo", "Bar");
+
+ // expected in response
+ // status code, status message, all headers from server response --> ff attributes
+ // server response message body into payload of ff
+ final MockFlowFile bundle1 = runner.getFlowFilesForRelationship(InvokeHTTP.REL_RESPONSE).get(0);
+ bundle1.assertContentEquals("NO".getBytes("UTF-8"));
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "404");
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "Not Found");
+ bundle1.assertAttributeEquals("Foo", "Bar");
+ bundle1.assertAttributeEquals("Content-Type", "text/plain; charset=ISO-8859-1");
+ }
+
+ @Test
+ public void testOutputResponseRegardlessWithOutputInAttributeLarge() throws Exception {
+ addHandler(new GetLargeHandler());
+
+ runner.setProperty(InvokeHTTP.PROP_URL, url + "/status/200");
+ runner.setProperty(InvokeHTTP.PROP_METHOD, "POST");
+ runner.setProperty(InvokeHTTP.PROP_OUTPUT_RESPONSE_REGARDLESS,"true");
+ runner.setProperty(InvokeHTTP.PROP_PUT_OUTPUT_IN_ATTRIBUTE,"outputBody");
+ runner.setProperty(InvokeHTTP.PROP_PUT_ATTRIBUTE_MAX_LENGTH,"11");
+
+ createFlowFiles(runner);
+
+ runner.run();
+
+ runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
+
+ // expected in request status.code and status.message
+ // original flow file (+attributes)
+ final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.REL_NO_RETRY).get(0);
+ bundle.assertContentEquals("Hello".getBytes("UTF-8"));
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "404");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "Not Found");
+ bundle.assertAttributeEquals("outputBody", "Lorem ipsum");
+ bundle.assertAttributeEquals("Foo", "Bar");
+
+ // expected in response
+ // status code, status message, all headers from server response --> ff attributes
+ // server response message body into payload of ff
+ final MockFlowFile bundle1 = runner.getFlowFilesForRelationship(InvokeHTTP.REL_RESPONSE).get(0);
+ bundle1.assertContentEquals("Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. "
+ + "Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor "
+ + "in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, "
+ + "sunt in culpa qui officia deserunt mollit anim id est laborum.");
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "404");
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "Not Found");
+ bundle1.assertAttributeEquals("Foo", "Bar");
+ bundle1.assertAttributeEquals("Content-Type", "text/plain; charset=ISO-8859-1");
+ }
+
+
+ @Test
+ public void testMultipleSameHeaders() throws Exception {
+ addHandler(new GetMultipleHeaderHandler());
+
+ runner.setProperty(InvokeHTTP.PROP_URL, url + "/status/200");
+
+ createFlowFiles(runner);
+
+ runner.run();
+
+ runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
+
+ // expected in request status.code and status.message
+ // original flow file (+attributes)
+ final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.REL_SUCCESS_REQ).get(0);
+ bundle.assertContentEquals("Hello".getBytes("UTF-8"));
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
+ bundle.assertAttributeEquals("Foo", "Bar");
+
+ // expected in response
+ // status code, status message, all headers from server response --> ff attributes
+ // server response message body into payload of ff
+ final MockFlowFile bundle1 = runner.getFlowFilesForRelationship(InvokeHTTP.REL_RESPONSE).get(0);
+ bundle1.assertContentEquals("/status/200".getBytes("UTF-8"));
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
+ bundle1.assertAttributeEquals("Foo", "Bar");
+ bundle1.assertAttributeEquals("double", "1, 2");
+ bundle1.assertAttributeEquals("Content-Type", "text/plain; charset=ISO-8859-1");
+ }
+
+ @Test
+ public void testPutResponseHeadersInRequest() throws Exception {
+ addHandler(new GetMultipleHeaderHandler());
+
+ runner.setProperty(InvokeHTTP.PROP_URL, url + "/status/200");
+ runner.setProperty(InvokeHTTP.PROP_ADD_HEADERS_TO_REQUEST, "true");
+
+ createFlowFiles(runner);
+
+ runner.run();
+
+ runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
+
+ // expected in request status.code and status.message
+ // original flow file (+all attributes from response)
+ final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.REL_SUCCESS_REQ).get(0);
+ bundle.assertContentEquals("Hello".getBytes("UTF-8"));
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
+ bundle.assertAttributeEquals("Foo", "Bar");
+ bundle.assertAttributeEquals("double", "1, 2");
+ bundle.assertAttributeEquals("Content-Type", "text/plain; charset=ISO-8859-1");
+
+ // expected in response
+ // status code, status message, all headers from server response --> ff attributes
+ // server response message body into payload of ff
+ final MockFlowFile bundle1 = runner.getFlowFilesForRelationship(InvokeHTTP.REL_RESPONSE).get(0);
+ bundle1.assertContentEquals("/status/200".getBytes("UTF-8"));
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
+ bundle1.assertAttributeEquals("Foo", "Bar");
+ bundle1.assertAttributeEquals("double", "1, 2");
+ bundle1.assertAttributeEquals("Content-Type", "text/plain; charset=ISO-8859-1");
+ }
+
+ @Test
+ public void testToRequestAttribute() throws Exception {
+ addHandler(new GetOrHeadHandler());
+
+ runner.setProperty(InvokeHTTP.PROP_URL, url + "/status/200");
+ runner.setProperty(InvokeHTTP.PROP_PUT_OUTPUT_IN_ATTRIBUTE,"outputBody");
+
+ createFlowFiles(runner);
+
+ runner.run();
+
+ runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
+
+ // expected in request status.code, status.message and body of response in attribute
+ // original flow file (+attributes)
+ final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.REL_SUCCESS_REQ).get(0);
+ bundle.assertContentEquals("Hello".getBytes("UTF-8"));
+ bundle.assertAttributeEquals("outputBody", "/status/200");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
+ bundle.assertAttributeEquals("Foo", "Bar");
+ }
+
+ @Test
+ public void testNoInput() throws Exception {
+ addHandler(new GetOrHeadHandler());
+
+ runner.setProperty(InvokeHTTP.PROP_URL, url + "/status/200");
+ runner.setProperty(InvokeHTTP.PROP_METHOD,"GET");
+ runner.setIncomingConnection(false);
+ runner.setNonLoopConnection(false);
+
+ runner.run();
+
+ runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
+
+ // expected in response
+ // status code, status message, all headers from server response --> ff attributes
+ // server response message body into payload of ff
+ final MockFlowFile bundle1 = runner.getFlowFilesForRelationship(InvokeHTTP.REL_RESPONSE).get(0);
+ bundle1.assertContentEquals("/status/200".getBytes("UTF-8"));
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
+ bundle1.assertAttributeEquals("Content-Type", "text/plain; charset=ISO-8859-1");
+ }
+
+ @Test
+ public void testNoInputFail() throws Exception {
+ addHandler(new GetOrHeadHandler());
+
+ runner.setProperty(InvokeHTTP.PROP_URL, url + "/status/200");
+ runner.setProperty(InvokeHTTP.PROP_METHOD, "POST");
+ runner.setIncomingConnection(false);
+ runner.setNonLoopConnection(false);
+
+ runner.run();
+
+ runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
+
+ runner.setProperty(InvokeHTTP.PROP_METHOD,"OPTION");
+
+ runner.run();
+
+ runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
+ }
+
+ @Test
+ public void testNoInputSendToAttribute() throws Exception {
+ addHandler(new GetOrHeadHandler());
+
+ runner.setProperty(InvokeHTTP.PROP_URL, url + "/status/200");
+ runner.setProperty(InvokeHTTP.PROP_PUT_OUTPUT_IN_ATTRIBUTE, "outputBody");
+ runner.setIncomingConnection(false);
+ runner.setNonLoopConnection(false);
+
+ runner.run();
+
+ runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
+
+ // expected in request
+ // status code, status message, no ff content
+ // server response message body into attribute of ff
+ final MockFlowFile bundle1 = runner.getFlowFilesForRelationship(InvokeHTTP.REL_SUCCESS_REQ).get(0);
+ bundle1.assertContentEquals("".getBytes("UTF-8"));
+ bundle1.assertAttributeEquals("outputBody", "/status/200");
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
+ }
+
+ @Test
public void test200Auth() throws Exception {
addHandler(new BasicAuthHandler());
final String username = "basic_user";
final String password = "basic_password";
- runner.setProperty(InvokeHTTP.Config.PROP_URL, url + "/status/200");
- runner.setProperty(InvokeHTTP.Config.PROP_BASIC_AUTH_USERNAME, username);
- runner.setProperty(InvokeHTTP.Config.PROP_BASIC_AUTH_PASSWORD, password);
+ runner.setProperty(InvokeHTTP.PROP_URL, url + "/status/200");
+ runner.setProperty(InvokeHTTP.PROP_BASIC_AUTH_USERNAME, username);
+ runner.setProperty(InvokeHTTP.PROP_BASIC_AUTH_PASSWORD, password);
final byte[] creds = String.format("%s:%s", username, password).getBytes(StandardCharsets.UTF_8);
final String expAuth = String.format("Basic %s", new String(encodeBase64(creds)));
@@ -142,28 +470,28 @@ public abstract class TestInvokeHttpCommon {
runner.run();
- runner.assertTransferCount(InvokeHTTP.Config.REL_SUCCESS_REQ, 1);
- runner.assertTransferCount(InvokeHTTP.Config.REL_SUCCESS_RESP, 1);
- runner.assertTransferCount(InvokeHTTP.Config.REL_RETRY, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_NO_RETRY, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_FAILURE, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
// expected in request status.code and status.message
// original flow file (+attributes)
- final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.Config.REL_SUCCESS_REQ).get(0);
+ final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.REL_SUCCESS_REQ).get(0);
bundle.assertContentEquals("Hello".getBytes("UTF-8"));
- bundle.assertAttributeEquals(InvokeHTTP.Config.STATUS_CODE, "200");
- bundle.assertAttributeEquals(InvokeHTTP.Config.STATUS_MESSAGE, "OK");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
bundle.assertAttributeEquals("Foo", "Bar");
// expected in response
// status code, status message, all headers from server response --> ff attributes
// server response message body into payload of ff
- final MockFlowFile bundle1 = runner.getFlowFilesForRelationship(InvokeHTTP.Config.REL_SUCCESS_RESP).get(0);
+ final MockFlowFile bundle1 = runner.getFlowFilesForRelationship(InvokeHTTP.REL_RESPONSE).get(0);
final String bundle1Content = new String(bundle1.toByteArray(), StandardCharsets.UTF_8);
assertTrue(bundle1Content.startsWith(expAuth)); // use startsWith instead of equals so we can ignore line endings
- bundle1.assertAttributeEquals(InvokeHTTP.Config.STATUS_CODE, "200");
- bundle1.assertAttributeEquals(InvokeHTTP.Config.STATUS_MESSAGE, "OK");
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
bundle1.assertAttributeEquals("Foo", "Bar");
bundle1.assertAttributeEquals("Content-Type", "text/plain; charset=ISO-8859-1");
@@ -190,55 +518,140 @@ public abstract class TestInvokeHttpCommon {
final String username = "basic_user";
final String password = "basic_password";
- runner.setProperty(InvokeHTTP.Config.PROP_URL, url + "/status/401");
- runner.setProperty(InvokeHTTP.Config.PROP_BASIC_AUTH_USERNAME, username);
- runner.setProperty(InvokeHTTP.Config.PROP_BASIC_AUTH_PASSWORD, password);
+ runner.setProperty(InvokeHTTP.PROP_URL, url + "/status/401");
+ runner.setProperty(InvokeHTTP.PROP_BASIC_AUTH_USERNAME, username);
+ runner.setProperty(InvokeHTTP.PROP_BASIC_AUTH_PASSWORD, password);
createFlowFiles(runner);
runner.run();
- runner.assertTransferCount(InvokeHTTP.Config.REL_SUCCESS_REQ, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_SUCCESS_RESP, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_RETRY, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_NO_RETRY, 1);
- runner.assertTransferCount(InvokeHTTP.Config.REL_FAILURE, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
// expected in request status.code and status.message
// original flow file (+attributes)
- final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.Config.REL_NO_RETRY).get(0);
- bundle.assertAttributeEquals(InvokeHTTP.Config.STATUS_CODE, "401");
- bundle.assertAttributeEquals(InvokeHTTP.Config.STATUS_MESSAGE, "Unauthorized");
+ final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.REL_NO_RETRY).get(0);
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "401");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "Unauthorized");
+ bundle.assertAttributeEquals("Foo", "Bar");
+ final String actual = new String(bundle.toByteArray(), StandardCharsets.UTF_8);
+ final String expected = "Hello";
+ Assert.assertEquals(expected, actual);
+
+ final String response = bundle.getAttribute(InvokeHTTP.RESPONSE_BODY);
+ assertEquals(response, "Get off my lawn!"+System.lineSeparator());
+ }
+
+ @Test
+ public void test200DigestAuth() throws Exception {
+ addHandler(new DigestAuthHandler());
+ final String username = "basic_user";
+ final String password = "basic_password";
+
+ runner.setProperty(InvokeHTTP.PROP_URL, url + "/status/200");
+ runner.setProperty(InvokeHTTP.PROP_BASIC_AUTH_USERNAME, username);
+ runner.setProperty(InvokeHTTP.PROP_BASIC_AUTH_PASSWORD, password);
+ runner.setProperty(InvokeHTTP.PROP_DIGEST_AUTH,"true");
+
+ createFlowFiles(runner);
+
+ runner.run();
+
+ runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
+
+ //expected in request status.code and status.message
+ //original flow file (+attributes)
+ final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.REL_SUCCESS_REQ).get(0);
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
+ bundle.assertAttributeEquals("Foo", "Bar");
+ final String actual = new String(bundle.toByteArray(), StandardCharsets.UTF_8);
+ final String expected = "Hello";
+ Assert.assertEquals(expected, actual);
+
+ //expected in response
+ //status code, status message, all headers from server response --> ff attributes
+ //server response message body into payload of ff
+ final MockFlowFile bundle1 = runner.getFlowFilesForRelationship(InvokeHTTP.REL_RESPONSE).get(0);
+ bundle1.assertContentEquals(("DIGEST"+System.lineSeparator()).getBytes("UTF-8"));
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
+ bundle1.assertAttributeEquals("Foo", "Bar");
+ bundle1.assertAttributeEquals("Content-Type", "text/plain; charset=ISO-8859-1");
+ }
+
+ @Test
+ public void test401DigestNotAuth() throws Exception {
+ addHandler(new DigestAuthHandler());
+
+ runner.setProperty(InvokeHTTP.PROP_URL, url + "/status/200");
+ runner.setProperty(InvokeHTTP.PROP_DIGEST_AUTH,"false");
+ runner.setProperty(InvokeHTTP.PROP_PUT_ATTRIBUTE_MAX_LENGTH,"512");
+
+ createFlowFiles(runner);
+
+ runner.run();
+
+ runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
+
+ //expected in request status.code and status.message
+ //original flow file (+attributes)
+ final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.REL_NO_RETRY).get(0);
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "401");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "Unauthorized");
bundle.assertAttributeEquals("Foo", "Bar");
final String actual = new String(bundle.toByteArray(), StandardCharsets.UTF_8);
final String expected = "Hello";
Assert.assertEquals(expected, actual);
- final String response = bundle.getAttribute(InvokeHTTP.Config.RESPONSE_BODY);
- assertEquals(response, "Get off my lawn!");
+ final String response = bundle.getAttribute(InvokeHTTP.RESPONSE_BODY);
+ assertEquals("<html>\n" +
+ "<head>\n" +
+ "<meta http-equiv=\"Content-Type\" content=\"text/html;charset=ISO-8859-1\"/>\n" +
+ "<title>Error 401 </title>\n" +
+ "</head>\n" +
+ "<body>\n" +
+ "<h2>HTTP ERROR: 401</h2>\n" +
+ "<p>Problem accessing /status/200. Reason:\n" +
+ "<pre> Unauthorized</pre></p>\n" +
+ "<hr /><i><small>Powered by Jetty://</small></i>\n" +
+ "</body>\n" +
+ "</html>\n", response);
}
@Test
public void test500() throws Exception {
addHandler(new GetOrHeadHandler());
- runner.setProperty(InvokeHTTP.Config.PROP_URL, url + "/status/500");
+ runner.setProperty(InvokeHTTP.PROP_URL, url + "/status/500");
createFlowFiles(runner);
runner.run();
- runner.assertTransferCount(InvokeHTTP.Config.REL_SUCCESS_REQ, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_SUCCESS_RESP, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_RETRY, 1);
- runner.assertTransferCount(InvokeHTTP.Config.REL_NO_RETRY, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_FAILURE, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_RETRY, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
// expected in response
- final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.Config.REL_RETRY).get(0);
+ final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.REL_RETRY).get(0);
final String actual = new String(bundle.toByteArray(), StandardCharsets.UTF_8);
- bundle.assertAttributeEquals(InvokeHTTP.Config.STATUS_CODE, "500");
- bundle.assertAttributeEquals(InvokeHTTP.Config.STATUS_MESSAGE, "Server Error");
- bundle.assertAttributeEquals(InvokeHTTP.Config.RESPONSE_BODY, "/status/500");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "500");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "Server Error");
+ bundle.assertAttributeEquals(InvokeHTTP.RESPONSE_BODY, "/status/500");
final String expected = "Hello";
Assert.assertEquals(expected, actual);
@@ -250,23 +663,23 @@ public abstract class TestInvokeHttpCommon {
public void test300() throws Exception {
addHandler(new GetOrHeadHandler());
- runner.setProperty(InvokeHTTP.Config.PROP_URL, url + "/status/302");
+ runner.setProperty(InvokeHTTP.PROP_URL, url + "/status/302");
createFlowFiles(runner);
runner.run();
- runner.assertTransferCount(InvokeHTTP.Config.REL_SUCCESS_REQ, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_SUCCESS_RESP, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_RETRY, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_NO_RETRY, 1);
- runner.assertTransferCount(InvokeHTTP.Config.REL_FAILURE, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
// getMyFlowFiles();
// expected in response
- final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.Config.REL_NO_RETRY).get(0);
+ final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.REL_NO_RETRY).get(0);
final String actual = new String(bundle.toByteArray(), StandardCharsets.UTF_8);
- bundle.assertAttributeEquals(InvokeHTTP.Config.STATUS_CODE, "302");
- bundle.assertAttributeEquals(InvokeHTTP.Config.STATUS_MESSAGE, "Found");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "302");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "Found");
final String expected = "Hello";
Assert.assertEquals(expected, actual);
bundle.assertAttributeEquals("Foo", "Bar");
@@ -277,23 +690,23 @@ public abstract class TestInvokeHttpCommon {
public void test304() throws Exception {
addHandler(new GetOrHeadHandler());
- runner.setProperty(InvokeHTTP.Config.PROP_URL, url + "/status/304");
+ runner.setProperty(InvokeHTTP.PROP_URL, url + "/status/304");
createFlowFiles(runner);
runner.run();
- runner.assertTransferCount(InvokeHTTP.Config.REL_SUCCESS_REQ, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_SUCCESS_RESP, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_RETRY, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_NO_RETRY, 1);
- runner.assertTransferCount(InvokeHTTP.Config.REL_FAILURE, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
// getMyFlowFiles();
// expected in response
- final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.Config.REL_NO_RETRY).get(0);
+ final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.REL_NO_RETRY).get(0);
final String actual = new String(bundle.toByteArray(), StandardCharsets.UTF_8);
- bundle.assertAttributeEquals(InvokeHTTP.Config.STATUS_CODE, "304");
- bundle.assertAttributeEquals(InvokeHTTP.Config.STATUS_MESSAGE, "Not Modified");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "304");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "Not Modified");
final String expected = "Hello";
Assert.assertEquals(expected, actual);
bundle.assertAttributeEquals("Foo", "Bar");
@@ -304,24 +717,24 @@ public abstract class TestInvokeHttpCommon {
public void test400() throws Exception {
addHandler(new GetOrHeadHandler());
- runner.setProperty(InvokeHTTP.Config.PROP_URL, url + "/status/400");
+ runner.setProperty(InvokeHTTP.PROP_URL, url + "/status/400");
createFlowFiles(runner);
runner.run();
- runner.assertTransferCount(InvokeHTTP.Config.REL_SUCCESS_REQ, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_SUCCESS_RESP, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_RETRY, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_NO_RETRY, 1);
- runner.assertTransferCount(InvokeHTTP.Config.REL_FAILURE, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
// getMyFlowFiles();
// expected in response
- final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.Config.REL_NO_RETRY).get(0);
+ final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.REL_NO_RETRY).get(0);
final String actual = new String(bundle.toByteArray(), StandardCharsets.UTF_8);
- bundle.assertAttributeEquals(InvokeHTTP.Config.STATUS_CODE, "400");
- bundle.assertAttributeEquals(InvokeHTTP.Config.STATUS_MESSAGE, "Bad Request");
- bundle.assertAttributeEquals(InvokeHTTP.Config.RESPONSE_BODY, "/status/400");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "400");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "Bad Request");
+ bundle.assertAttributeEquals(InvokeHTTP.RESPONSE_BODY, "/status/400");
final String expected = "Hello";
Assert.assertEquals(expected, actual);
bundle.assertAttributeEquals("Foo", "Bar");
@@ -332,25 +745,25 @@ public abstract class TestInvokeHttpCommon {
public void test412() throws Exception {
addHandler(new GetOrHeadHandler());
- runner.setProperty(InvokeHTTP.Config.PROP_URL, url + "/status/412");
- runner.setProperty(InvokeHTTP.Config.PROP_METHOD, "GET");
+ runner.setProperty(InvokeHTTP.PROP_URL, url + "/status/412");
+ runner.setProperty(InvokeHTTP.PROP_METHOD, "GET");
createFlowFiles(runner);
runner.run();
- runner.assertTransferCount(InvokeHTTP.Config.REL_SUCCESS_REQ, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_SUCCESS_RESP, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_RETRY, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_NO_RETRY, 1);
- runner.assertTransferCount(InvokeHTTP.Config.REL_FAILURE, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
// expected in response
- final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.Config.REL_NO_RETRY).get(0);
+ final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.REL_NO_RETRY).get(0);
final String actual = new String(bundle.toByteArray(), StandardCharsets.UTF_8);
- bundle.assertAttributeEquals(InvokeHTTP.Config.STATUS_CODE, "412");
- bundle.assertAttributeEquals(InvokeHTTP.Config.STATUS_MESSAGE, "Precondition Failed");
- bundle.assertAttributeEquals(InvokeHTTP.Config.RESPONSE_BODY, "/status/412");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "412");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "Precondition Failed");
+ bundle.assertAttributeEquals(InvokeHTTP.RESPONSE_BODY, "/status/412");
final String expected = "Hello";
Assert.assertEquals(expected, actual);
bundle.assertAttributeEquals("Foo", "Bar");
@@ -361,28 +774,28 @@ public abstract class TestInvokeHttpCommon {
public void testHead() throws Exception {
addHandler(new GetOrHeadHandler());
- runner.setProperty(InvokeHTTP.Config.PROP_METHOD, "HEAD");
- runner.setProperty(InvokeHTTP.Config.PROP_URL, url + "/status/200");
+ runner.setProperty(InvokeHTTP.PROP_METHOD, "HEAD");
+ runner.setProperty(InvokeHTTP.PROP_URL, url + "/status/200");
createFlowFiles(runner);
runner.run();
- runner.assertTransferCount(InvokeHTTP.Config.REL_SUCCESS_REQ, 1);
- runner.assertTransferCount(InvokeHTTP.Config.REL_SUCCESS_RESP, 1);
- runner.assertTransferCount(InvokeHTTP.Config.REL_RETRY, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_NO_RETRY, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_FAILURE, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
- final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.Config.REL_SUCCESS_REQ).get(0);
+ final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.REL_SUCCESS_REQ).get(0);
bundle.assertContentEquals("Hello".getBytes("UTF-8"));
- bundle.assertAttributeEquals(InvokeHTTP.Config.STATUS_CODE, "200");
- bundle.assertAttributeEquals(InvokeHTTP.Config.STATUS_MESSAGE, "OK");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
bundle.assertAttributeEquals("Foo", "Bar");
- final MockFlowFile bundle1 = runner.getFlowFilesForRelationship(InvokeHTTP.Config.REL_SUCCESS_RESP).get(0);
+ final MockFlowFile bundle1 = runner.getFlowFilesForRelationship(InvokeHTTP.REL_RESPONSE).get(0);
bundle1.assertContentEquals("".getBytes("UTF-8"));
- bundle1.assertAttributeEquals(InvokeHTTP.Config.STATUS_CODE, "200");
- bundle1.assertAttributeEquals(InvokeHTTP.Config.STATUS_MESSAGE, "OK");
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
bundle1.assertAttributeEquals("Foo", "Bar");
bundle1.assertAttributeEquals("Content-Type", "text/plain");
final String actual1 = new String(bundle1.toByteArray(), StandardCharsets.UTF_8);
@@ -394,28 +807,28 @@ public abstract class TestInvokeHttpCommon {
public void testPost() throws Exception {
addHandler(new PostHandler());
- runner.setProperty(InvokeHTTP.Config.PROP_METHOD, "POST");
- runner.setProperty(InvokeHTTP.Config.PROP_URL, url + "/post");
+ runner.setProperty(InvokeHTTP.PROP_METHOD, "POST");
+ runner.setProperty(InvokeHTTP.PROP_URL, url + "/post");
createFlowFiles(runner);
runner.run();
- runner.assertTransferCount(InvokeHTTP.Config.REL_SUCCESS_REQ, 1);
- runner.assertTransferCount(InvokeHTTP.Config.REL_SUCCESS_RESP, 1);
- runner.assertTransferCount(InvokeHTTP.Config.REL_RETRY, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_NO_RETRY, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_FAILURE, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
- final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.Config.REL_SUCCESS_REQ).get(0);
+ final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.REL_SUCCESS_REQ).get(0);
bundle.assertContentEquals("Hello".getBytes("UTF-8"));
- bundle.assertAttributeEquals(InvokeHTTP.Config.STATUS_CODE, "200");
- bundle.assertAttributeEquals(InvokeHTTP.Config.STATUS_MESSAGE, "OK");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
bundle.assertAttributeEquals("Foo", "Bar");
- final MockFlowFile bundle1 = runner.getFlowFilesForRelationship(InvokeHTTP.Config.REL_SUCCESS_RESP).get(0);
+ final MockFlowFile bundle1 = runner.getFlowFilesForRelationship(InvokeHTTP.REL_RESPONSE).get(0);
bundle1.assertContentEquals("".getBytes("UTF-8"));
- bundle1.assertAttributeEquals(InvokeHTTP.Config.STATUS_CODE, "200");
- bundle1.assertAttributeEquals(InvokeHTTP.Config.STATUS_MESSAGE, "OK");
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
bundle1.assertAttributeEquals("Foo", "Bar");
bundle1.assertAttributeNotExists("Content-Type");
@@ -426,30 +839,30 @@ public abstract class TestInvokeHttpCommon {
@Test
public void testPut() throws Exception {
- addHandler(new PostHandler());
+ addHandler(new PutHandler());
- runner.setProperty(InvokeHTTP.Config.PROP_METHOD, "PUT");
- runner.setProperty(InvokeHTTP.Config.PROP_URL, url + "/post");
+ runner.setProperty(InvokeHTTP.PROP_METHOD, "PUT");
+ runner.setProperty(InvokeHTTP.PROP_URL, url + "/post");
createFlowFiles(runner);
runner.run();
- runner.assertTransferCount(InvokeHTTP.Config.REL_SUCCESS_REQ, 1);
- runner.assertTransferCount(InvokeHTTP.Config.REL_SUCCESS_RESP, 1);
- runner.assertTransferCount(InvokeHTTP.Config.REL_RETRY, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_NO_RETRY, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_FAILURE, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
- final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.Config.REL_SUCCESS_REQ).get(0);
+ final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.REL_SUCCESS_REQ).get(0);
bundle.assertContentEquals("Hello".getBytes("UTF-8"));
- bundle.assertAttributeEquals(InvokeHTTP.Config.STATUS_CODE, "200");
- bundle.assertAttributeEquals(InvokeHTTP.Config.STATUS_MESSAGE, "OK");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
bundle.assertAttributeEquals("Foo", "Bar");
- final MockFlowFile bundle1 = runner.getFlowFilesForRelationship(InvokeHTTP.Config.REL_SUCCESS_RESP).get(0);
+ final MockFlowFile bundle1 = runner.getFlowFilesForRelationship(InvokeHTTP.REL_RESPONSE).get(0);
bundle1.assertContentEquals("".getBytes("UTF-8"));
- bundle1.assertAttributeEquals(InvokeHTTP.Config.STATUS_CODE, "200");
- bundle1.assertAttributeEquals(InvokeHTTP.Config.STATUS_MESSAGE, "OK");
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
bundle1.assertAttributeEquals("Foo", "Bar");
bundle1.assertAttributeNotExists("Content-Type");
@@ -462,28 +875,28 @@ public abstract class TestInvokeHttpCommon {
public void testDelete() throws Exception {
addHandler(new DeleteHandler());
- runner.setProperty(InvokeHTTP.Config.PROP_METHOD, "DELETE");
- runner.setProperty(InvokeHTTP.Config.PROP_URL, url + "/status/200");
+ runner.setProperty(InvokeHTTP.PROP_METHOD, "DELETE");
+ runner.setProperty(InvokeHTTP.PROP_URL, url + "/status/200");
createFlowFiles(runner);
runner.run();
- runner.assertTransferCount(InvokeHTTP.Config.REL_SUCCESS_REQ, 1);
- runner.assertTransferCount(InvokeHTTP.Config.REL_SUCCESS_RESP, 1);
- runner.assertTransferCount(InvokeHTTP.Config.REL_RETRY, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_NO_RETRY, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_FAILURE, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
- final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.Config.REL_SUCCESS_REQ).get(0);
+ final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.REL_SUCCESS_REQ).get(0);
bundle.assertContentEquals("Hello".getBytes("UTF-8"));
- bundle.assertAttributeEquals(InvokeHTTP.Config.STATUS_CODE, "200");
- bundle.assertAttributeEquals(InvokeHTTP.Config.STATUS_MESSAGE, "OK");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
bundle.assertAttributeEquals("Foo", "Bar");
- final MockFlowFile bundle1 = runner.getFlowFilesForRelationship(InvokeHTTP.Config.REL_SUCCESS_RESP).get(0);
+ final MockFlowFile bundle1 = runner.getFlowFilesForRelationship(InvokeHTTP.REL_RESPONSE).get(0);
bundle1.assertContentEquals("".getBytes("UTF-8"));
- bundle1.assertAttributeEquals(InvokeHTTP.Config.STATUS_CODE, "200");
- bundle1.assertAttributeEquals(InvokeHTTP.Config.STATUS_MESSAGE, "OK");
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
bundle1.assertAttributeEquals("Foo", "Bar");
final String actual1 = new String(bundle1.toByteArray(), StandardCharsets.UTF_8);
final String expected1 = "";
@@ -494,28 +907,28 @@ public abstract class TestInvokeHttpCommon {
public void testOptions() throws Exception {
addHandler(new OptionsHandler());
- runner.setProperty(InvokeHTTP.Config.PROP_METHOD, "OPTIONS");
- runner.setProperty(InvokeHTTP.Config.PROP_URL, url + "/status/200");
+ runner.setProperty(InvokeHTTP.PROP_METHOD, "OPTIONS");
+ runner.setProperty(InvokeHTTP.PROP_URL, url + "/status/200");
createFlowFiles(runner);
runner.run();
- runner.assertTransferCount(InvokeHTTP.Config.REL_SUCCESS_REQ, 1);
- runner.assertTransferCount(InvokeHTTP.Config.REL_SUCCESS_RESP, 1);
- runner.assertTransferCount(InvokeHTTP.Config.REL_RETRY, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_NO_RETRY, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_FAILURE, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
- final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.Config.REL_SUCCESS_REQ).get(0);
+ final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.REL_SUCCESS_REQ).get(0);
bundle.assertContentEquals("Hello".getBytes("UTF-8"));
- bundle.assertAttributeEquals(InvokeHTTP.Config.STATUS_CODE, "200");
- bundle.assertAttributeEquals(InvokeHTTP.Config.STATUS_MESSAGE, "OK");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
bundle.assertAttributeEquals("Foo", "Bar");
- final MockFlowFile bundle1 = runner.getFlowFilesForRelationship(InvokeHTTP.Config.REL_SUCCESS_RESP).get(0);
+ final MockFlowFile bundle1 = runner.getFlowFilesForRelationship(InvokeHTTP.REL_RESPONSE).get(0);
bundle1.assertContentEquals("/status/200".getBytes("UTF-8"));
- bundle1.assertAttributeEquals(InvokeHTTP.Config.STATUS_CODE, "200");
- bundle1.assertAttributeEquals(InvokeHTTP.Config.STATUS_MESSAGE, "OK");
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
bundle1.assertAttributeEquals("Foo", "Bar");
}
@@ -523,34 +936,36 @@ public abstract class TestInvokeHttpCommon {
public void testSendAttributes() throws Exception {
addHandler(new AttributesSentHandler());
- runner.setProperty(InvokeHTTP.Config.PROP_URL, url + "/status/200");
- runner.setProperty(InvokeHTTP.Config.PROP_ATTRIBUTES_TO_SEND, "Foo");
+ runner.setProperty(InvokeHTTP.PROP_URL, url + "/status/200");
+ runner.setProperty(InvokeHTTP.PROP_ATTRIBUTES_TO_SEND, "F.*");
+ runner.setProperty("dynamicHeader","yes!");
createFlowFiles(runner);
runner.run();
- runner.assertTransferCount(InvokeHTTP.Config.REL_SUCCESS_REQ, 1);
- runner.assertTransferCount(InvokeHTTP.Config.REL_SUCCESS_RESP, 1);
- runner.assertTransferCount(InvokeHTTP.Config.REL_RETRY, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_NO_RETRY, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_FAILURE, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
//expected in request status.code and status.message
//original flow file (+attributes)
- final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.Config.REL_SUCCESS_REQ).get(0);
+ final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.REL_SUCCESS_REQ).get(0);
bundle.assertContentEquals("Hello".getBytes("UTF-8"));
- bundle.assertAttributeEquals(InvokeHTTP.Config.STATUS_CODE, "200");
- bundle.assertAttributeEquals(InvokeHTTP.Config.STATUS_MESSAGE, "OK");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
bundle.assertAttributeEquals("Foo", "Bar");
//expected in response
//status code, status message, all headers from server response --> ff attributes
//server response message body into payload of ff
- final MockFlowFile bundle1 = runner.getFlowFilesForRelationship(InvokeHTTP.Config.REL_SUCCESS_RESP).get(0);
+ final MockFlowFile bundle1 = runner.getFlowFilesForRelationship(InvokeHTTP.REL_RESPONSE).get(0);
bundle1.assertContentEquals("Bar".getBytes("UTF-8"));
- bundle1.assertAttributeEquals(InvokeHTTP.Config.STATUS_CODE, "200");
- bundle1.assertAttributeEquals(InvokeHTTP.Config.STATUS_MESSAGE, "OK");
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
+ bundle1.assertAttributeEquals("dynamicHeader","yes!");
bundle1.assertAttributeEquals("Foo", "Bar");
bundle1.assertAttributeEquals("Content-Type", "text/plain; charset=ISO-8859-1");
}
@@ -559,19 +974,19 @@ public abstract class TestInvokeHttpCommon {
public void testReadTimeout() throws Exception {
addHandler(new ReadTimeoutHandler());
- runner.setProperty(InvokeHTTP.Config.PROP_URL, url + "/status/200");
- runner.setProperty(InvokeHTTP.Config.PROP_READ_TIMEOUT, "5 secs");
+ runner.setProperty(InvokeHTTP.PROP_URL, url + "/status/200");
+ runner.setProperty(InvokeHTTP.PROP_READ_TIMEOUT, "5 secs");
createFlowFiles(runner);
runner.run();
- runner.assertTransferCount(InvokeHTTP.Config.REL_SUCCESS_REQ, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_SUCCESS_RESP, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_RETRY, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_NO_RETRY, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_FAILURE, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 1);
- final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.Config.REL_FAILURE).get(0);
+ final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.REL_FAILURE).get(0);
final String actual = new String(bundle.toByteArray(), StandardCharsets.UTF_8);
final String expected = "Hello";
@@ -586,17 +1001,17 @@ public abstract class TestInvokeHttpCommon {
// this is the bad urls
final String badurlport = "http://localhost:" + 445;
- runner.setProperty(InvokeHTTP.Config.PROP_URL, badurlport + "/doesnotExist");
+ runner.setProperty(InvokeHTTP.PROP_URL, badurlport + "/doesnotExist");
createFlowFiles(runner);
runner.run();
- runner.assertTransferCount(InvokeHTTP.Config.REL_SUCCESS_REQ, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_SUCCESS_RESP, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_RETRY, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_NO_RETRY, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_FAILURE, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 1);
- final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.Config.REL_FAILURE).get(0);
+ final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.REL_FAILURE).get(0);
final String actual = new String(bundle.toByteArray(), StandardCharsets.UTF_8);
final String expected = "Hello";
@@ -610,17 +1025,17 @@ public abstract class TestInvokeHttpCommon {
final String badurlhost = "http://localhOOst:" + 445;
- runner.setProperty(InvokeHTTP.Config.PROP_URL, badurlhost + "/doesnotExist");
+ runner.setProperty(InvokeHTTP.PROP_URL, badurlhost + "/doesnotExist");
createFlowFiles(runner);
runner.run();
- runner.assertTransferCount(InvokeHTTP.Config.REL_SUCCESS_REQ, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_SUCCESS_RESP, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_RETRY, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_NO_RETRY, 0);
- runner.assertTransferCount(InvokeHTTP.Config.REL_FAILURE, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 1);
- final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.Config.REL_FAILURE).get(0);
+ final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.REL_FAILURE).get(0);
final String actual = new String(bundle.toByteArray(), StandardCharsets.UTF_8);
final String expected = "Hello";
@@ -628,6 +1043,46 @@ public abstract class TestInvokeHttpCommon {
bundle.assertAttributeEquals("Foo", "Bar");
}
+ @Test
+ public void testArbitraryRequest() throws Exception {
+ addHandler(new FetchHandler());
+
+ runner.setProperty(InvokeHTTP.PROP_URL, url + "/status/200");
+ runner.setProperty(InvokeHTTP.PROP_METHOD,"FETCH");
+
+ createFlowFiles(runner);
+
+ runner.run();
+
+ runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 1);
+ runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 0);
+ runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
+
+ //expected in request status.code and status.message
+ //original flow file (+attributes)
+ final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.REL_SUCCESS_REQ).get(0);
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
+ bundle.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
+ final String actual = new String(bundle.toByteArray(), StandardCharsets.UTF_8);
+ final String expected = "Hello";
+ Assert.assertEquals(expected, actual);
+ bundle.assertAttributeEquals("Foo", "Bar");
+
+ //expected in response
+ //status code, status message, all headers from server response --> ff attributes
+ //server response message body into payload of ff
+ final MockFlowFile bundle1 = runner.getFlowFilesForRelationship(InvokeHTTP.REL_RESPONSE).get(0);
+ bundle1.assertContentEquals("/status/200".getBytes("UTF-8"));
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
+ bundle1.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
+ bundle1.assertAttributeEquals("Foo", "Bar");
+ bundle1.assertAttributeEquals("Content-Type", "text/plain; charset=ISO-8859-1");
+ final String actual1 = new String(bundle1.toByteArray(), StandardCharsets.UTF_8);
+ final String expected1 = "/status/200";
+ Assert.assertEquals(expected1, actual1);
+ }
public static void createFlowFiles(final TestRunner testRunner) throws UnsupportedEncodingException {
final Map<String, String> attributes = new HashMap<>();
@@ -638,7 +1093,7 @@ public abstract class TestInvokeHttpCommon {
}
- private static class DateHandler extends AbstractHandler {
+ protected static class DateHandler extends AbstractHandler {
private String dateString;
@@ -654,7 +1109,7 @@ public abstract class TestInvokeHttpCommon {
}
}
- private static class PostHandler extends AbstractHandler {
+ public static class PostHandler extends AbstractHandler {
@Override
public void handle(String target, Request baseRequest,
@@ -663,15 +1118,42 @@ public abstract class TestInvokeHttpCommon {
baseRequest.setHandled(true);
- assertEquals("/post", target);
+ if("POST".equals(request.getMethod())) {
+ assertEquals("application/plain-text",request.getHeader("Content-Type"));
+ final String body = request.getReader().readLine();
+ assertEquals("Hello", body);
+ } else {
+ response.setStatus(404);
+ response.setContentType("text/plain");
+ response.setContentLength(0);
+ }
+
+ }
+ }
- final String body = request.getReader().readLine();
- assertEquals("Hello", body);
+ public static class PutHandler extends AbstractHandler {
+
+ @Override
+ public void handle(String target, Request baseRequest,
+ HttpServletRequest request, HttpServletResponse response)
+ throws IOException, ServletException {
+
+ baseRequest.setHandled(true);
+
+ if("PUT".equalsIgnoreCase(request.getMethod())) {
+ assertEquals("application/plain-text",request.getHeader("Content-Type"));
+ final String body = request.getReader().readLine();
+ assertEquals("Hello", body);
+ } else {
+ response.setStatus(404);
+ response.setContentType("text/plain");
+ response.setContentLength(0);
+ }
}
}
- private static class GetOrHeadHandler extends AbstractHandler {
+ public static class GetOrHeadHandler extends AbstractHandler {
@Override
public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
@@ -688,12 +1170,91 @@ public abstract class TestInvokeHttpCommon {
writer.print(target);
writer.flush();
}
+ } else if(!"HEAD".equalsIgnoreCase(request.getMethod())) {
+ response.setStatus(404);
+ response.setContentType("text/plain");
+ String body = "NO";
+ response.setContentLength(body.length());
+ response.setContentType("text/plain");
+
+ try (PrintWriter writer = response.getWriter()) {
+ writer.print(body);
+ writer.flush();
+ }
}
}
}
- private static class DeleteHandler extends AbstractHandler {
+ public static class GetLargeHandler extends AbstractHandler {
+
+ @Override
+ public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
+ baseRequest.setHandled(true);
+
+ final int status = Integer.valueOf(target.substring("/status".length() + 1));
+ response.setStatus(status);
+
+ response.setContentType("text/plain");
+ response.setContentLength(target.length());
+
+ if ("GET".equalsIgnoreCase(request.getMethod())) {
+ try (PrintWriter writer = response.getWriter()) {
+ writer.print(target);
+ writer.flush();
+ }
+ } else {
+ response.setStatus(404);
+ response.setContentType("text/plain");
+
+ //Lorem Ipsum
+ String body = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. "
+ + "Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor "
+ + "in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, "
+ + "sunt in culpa qui officia deserunt mollit anim id est laborum.";
+
+ response.setContentLength(body.length());
+ response.setContentType("text/plain");
+
+ try (PrintWriter writer = response.getWriter()) {
+ writer.print(body);
+ writer.flush();
+ }
+ }
+
+ }
+ }
+
+ public static class GetMultipleHeaderHandler extends AbstractHandler {
+
+ @Override
+ public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
+ baseRequest.setHandled(true);
+
+ final int status = Integer.valueOf(target.substring("/status".length() + 1));
+ response.setStatus(status);
+
+ response.setContentType("text/plain");
+ response.setContentLength(target.length());
+
+ if ("GET".equalsIgnoreCase(request.getMethod())) {
+ response.addHeader("double", "1");
+ response.addHeader("double", "2");
+
+ try (PrintWriter writer = response.getWriter()) {
+ writer.print(target);
+ writer.flush();
+ }
+ } else {
+ response.setStatus(404);
+ response.setContentType("text/plain");
+ response.setContentLength(0);
+ }
+
+ }
+ }
+
+ public static class DeleteHandler extends AbstractHandler {
@Override
public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
@@ -711,7 +1272,7 @@ public abstract class TestInvokeHttpCommon {
}
}
- private static class OptionsHandler extends AbstractHandler {
+ public static class OptionsHandler extends AbstractHandler {
@Override
public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
@@ -735,7 +1296,7 @@ public abstract class TestInvokeHttpCommon {
}
}
- private static class AttributesSentHandler extends AbstractHandler {
+ public static class AttributesSentHandler extends AbstractHandler {
@Override
public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
@@ -743,12 +1304,12 @@ public abstract class TestInvokeHttpCommon {
if ("Get".equalsIgnoreCase(request.getMethod())) {
String headerValue = request.getHeader("Foo");
+ response.setHeader("dynamicHeader",request.getHeader("dynamicHeader"));
final int status = Integer.valueOf(target.substring("/status".length() + 1));
response.setStatus(status);
response.setContentLength(headerValue.length());
response.setContentType("text/plain");
-
try (PrintWriter writer = response.getWriter()) {
writer.print(headerValue);
writer.flush();
@@ -761,7 +1322,7 @@ public abstract class TestInvokeHttpCommon {
}
}
- private static class ReadTimeoutHandler extends AbstractHandler {
+ public static class ReadTimeoutHandler extends AbstractHandler {
@Override
public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
@@ -792,9 +1353,7 @@ public abstract class TestInvokeHttpCommon {
}
}
-
-
- private static class BasicAuthHandler extends AbstractHandler {
+ public static class BasicAuthHandler extends AbstractHandler {
private String authString;
@@ -827,4 +1386,66 @@ public abstract class TestInvokeHttpCommon {
}
}
+ public static class DigestAuthHandler extends AbstractHandler {
+
+ private DigestAuthenticator digestAuthenticator;
+
+ private DigestAuthHandler() {
+ digestAuthenticator = new DigestAuthenticator();
+ ConstraintSecurityHandler securityHandler = new ConstraintSecurityHandler();
+
+ HashLoginService hashLoginService = new HashLoginService("realm");
+ hashLoginService.putUser("basic_user", new Password("basic_password"), new String[]{"realm"});
+ securityHandler.setLoginService(hashLoginService);
+ securityHandler.setIdentityService(new DefaultIdentityService());
+ digestAuthenticator.setConfiguration(securityHandler);
+ }
+
+
+ @Override
+ public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse
+ response)throws IOException, ServletException {
+ baseRequest.setHandled(true);
+
+ try {
+ Authentication authentication = digestAuthenticator.validateRequest(request, response, true);
+
+ if (authentication instanceof Authentication.User) {
+ response.setContentType("text/plain");
+ Authentication.User user = (Authentication.User) authentication;
+ response.getWriter().println(user.getAuthMethod());
+ } else if (authentication instanceof Authentication.ResponseSent) {
+ Authentication.ResponseSent responseSent = (Authentication.ResponseSent) authentication;
+ }
+ } catch (ServerAuthException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ public static class FetchHandler extends AbstractHandler {
+
+ @Override
+ public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
+ baseRequest.setHandled(true);
+
+
+ if ("Fetch".equalsIgnoreCase(request.getMethod())) {
+ final int status = Integer.valueOf(target.substring("/status".length() + 1));
+ response.setStatus(status);
+ response.setContentType("text/plain");
+ response.setContentLength(target.length());
+
+ try (PrintWriter writer = response.getWriter()) {
+ writer.print(target);
+ writer.flush();
+ }
+ } else {
+
+ response.setStatus(404);
+ response.setContentType("text/plain");
+ response.setContentLength(target.length());
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/8c2323dc/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 8a8cdb0..0708812 100644
--- a/pom.xml
+++ b/pom.xml
@@ -97,6 +97,44 @@
<hadoop.guava.version>12.0.1</hadoop.guava.version>
<yammer.metrics.version>2.2.0</yammer.metrics.version>
</properties>
+
+
+ <repositories>
+ <repository>
+ <id>central</id>
+ <!-- This should be at top, it makes maven try the central repo first and then others and hence faster dep resolution -->
+ <name>Maven Repository</name>
+ <url>https://repo1.maven.org/maven2</url>
+ <releases>
+ <enabled>true</enabled>
+ </releases>
+ <snapshots>
+ <enabled>false</enabled>
+ </snapshots>
+ </repository>
+ <repository>
+ <id>apache-repo</id>
+ <name>Apache Repository</name>
+ <url>https://repository.apache.org/content/repositories/releases</url>
+ <releases>
+ <enabled>true</enabled>
+ </releases>
+ <snapshots>
+ <enabled>false</enabled>
+ </snapshots>
+ </repository>
+ <repository>
+ <id>jcenter</id>
+ <url>http://jcenter.bintray.com </url>
+ <snapshots>
+ <enabled>false</enabled>
+ </snapshots>
+ <releases>
+ <enabled>true</enabled>
+ </releases>
+ </repository>
+ </repositories>
+
<dependencyManagement>
<dependencies>
<dependency>
[36/50] [abbrv] nifi git commit: NIFI-1210: Makes various changes to
the User Guide for the 0.4.0 version release.
Posted by mc...@apache.org.
NIFI-1210: Makes various changes to the User Guide for the 0.4.0 version release.
Reviewed and amended (see ticket for details) by Tony Kurc (tkurc@apache.org)
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/32734291
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/32734291
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/32734291
Branch: refs/heads/NIFI-655
Commit: 327342916e353b6c7e7fdc787375b4b30c42fde3
Parents: 1580edb
Author: Jenn Barnabee <je...@gmail.com>
Authored: Sat Nov 21 10:35:32 2015 -0500
Committer: Tony Kurc <tr...@gmail.com>
Committed: Sat Nov 21 22:57:19 2015 -0500
----------------------------------------------------------------------
.../asciidoc/images/nifi-connection-menu.png | Bin 0 -> 31189 bytes
.../main/asciidoc/images/nifi-connection.png | Bin 0 -> 166905 bytes
.../asciidoc/images/nifi-process-group-menu.png | Bin 0 -> 98964 bytes
.../asciidoc/images/nifi-processor-menu.png | Bin 0 -> 99962 bytes
.../src/main/asciidoc/images/nifi-rpg-menu.png | Bin 0 -> 108291 bytes
nifi-docs/src/main/asciidoc/user-guide.adoc | 105 +++++++++++++++++--
6 files changed, 94 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/32734291/nifi-docs/src/main/asciidoc/images/nifi-connection-menu.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/nifi-connection-menu.png b/nifi-docs/src/main/asciidoc/images/nifi-connection-menu.png
new file mode 100644
index 0000000..d78d99d
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/nifi-connection-menu.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/32734291/nifi-docs/src/main/asciidoc/images/nifi-connection.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/nifi-connection.png b/nifi-docs/src/main/asciidoc/images/nifi-connection.png
new file mode 100644
index 0000000..061f72b
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/nifi-connection.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/32734291/nifi-docs/src/main/asciidoc/images/nifi-process-group-menu.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/nifi-process-group-menu.png b/nifi-docs/src/main/asciidoc/images/nifi-process-group-menu.png
new file mode 100644
index 0000000..47d1f7e
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/nifi-process-group-menu.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/32734291/nifi-docs/src/main/asciidoc/images/nifi-processor-menu.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/nifi-processor-menu.png b/nifi-docs/src/main/asciidoc/images/nifi-processor-menu.png
new file mode 100644
index 0000000..d412aaf
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/nifi-processor-menu.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/32734291/nifi-docs/src/main/asciidoc/images/nifi-rpg-menu.png
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/images/nifi-rpg-menu.png b/nifi-docs/src/main/asciidoc/images/nifi-rpg-menu.png
new file mode 100644
index 0000000..c57dae6
Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/nifi-rpg-menu.png differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/32734291/nifi-docs/src/main/asciidoc/user-guide.adoc
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/user-guide.adoc b/nifi-docs/src/main/asciidoc/user-guide.adoc
index d914650..f8efa6f 100644
--- a/nifi-docs/src/main/asciidoc/user-guide.adoc
+++ b/nifi-docs/src/main/asciidoc/user-guide.adoc
@@ -40,7 +40,6 @@ use a supported web browser to view the User Interface. Supported web browsers i
Note that there is a known issue in Internet Explorer (IE) 10 and 11 that can cause problems when moving items on the NiFi graph. If you encounter this problem, we suggest using a browser other than IE. This known issue is described here: https://connect.microsoft.com/IE/Feedback/Details/1050422.
-
[template="glossary", id="terminology"]
Terminology
-----------
@@ -100,6 +99,11 @@ Terminology
As a result, several components may be combined together to make a larger building block from which to create a dataflow.
These templates can also be exported as XML and imported into another NiFi instance, allowing these building blocks to be shared.
+*flow.xml.gz*: Everything the DFM puts onto the NiFi User Interface canvas is written, in real time, to one file called the flow.xml.gz. This file is located in the nifi/conf directory.
+ Any change made on the canvas is automatically saved to this file, without the user needing to click a "save" button. In addition, the user may create a back-up copy of this file at any time
+ by selecting the Controller Settings button in the far-right section of the tool bar and clicking "Back-up flow" on the General tab. By default, this action saves a copy of the current flow in the nifi/conf/archive directory.
+ See <<Controller_Settings>> for a description of where the "Back-up flow" button may be found. (Note that in a NiFi Cluster, the NiFi Cluster Manager's copy of this file is named flow.tar, whereas this file is still named flow.xml.gz on the nodes.)
+
[[User_Interface]]
NiFi User Interface
@@ -191,7 +195,26 @@ image::add-processor-with-tag-cloud.png["Add Processor with Tag Cloud"]
Clicking the `Add` button or double-clicking on a Processor Type will add the selected Processor to the canvas at the
location that it was dropped.
-*Note*: For any component added to the graph, it is possible to select it with the mouse and move it anywhere on the graph. Also, it is possible to select multiple items at once by either holding down the Shift key and selecting each item or by holding down the Shift key and dragging a selection box around the desired components.
+*Note*: For any component added to the canvas, it is possible to select it with the mouse and move it anywhere on the graph. Also, it is possible to select multiple items at once by either holding down the Shift key and selecting each item or by holding down the Shift key and dragging a selection box around the desired components.
+
+Once a Processor has been dragged onto the canvas, the user may interact with it by right-clicking on the Processor and selecting an option from
+context menu.
+
+image::nifi-processor-menu.png["Processor Menu", width=300]
+
+The following options are available:
+
+- *Configure*: This option allows the user to establish or change the configuration of the Processor. (See <<Configuring_a_Processor>>.)
+- *Start* or *Stop*: This option allows the user to start or stop a Processor; the option will be either Start or Stop, depending on the current state of the Processor.
+- *Stats*: This option opens a graphical representation of the Processor's statistical information over time.
+- *Upstream connections*: This option allows the user to see and "jump to" upstream connections that are coming into the Processor. This is particularly useful when processors connect into and out of other Process Groups.
+- *Downstream connections*: This option allows the user to see and "jump to" downstream connections that are going out of the Processor. This is particularly useful when processors connect into and out of other Process Groups.
+- *Usage*: This option takes the user to the Processor's usage documentation.
+- *Change color*: This option allows the user to change the color of the Processor, which can make the visual management of large flows easier.
+- *Center in view*: This option centers the view of the canvas on the given Processor.
+- *Copy* or *Paste*: This option places a copy of the selected Processor on the clipboard, so that it may be pasted elsewhere on the canvas.
+- *Delete*: This option allows the DFM to delete a Processor from the canvas.
+
[[input_port]]
@@ -228,6 +251,24 @@ image:iconProcessGroup.png["Process Group", width=32]
and maintain. When a Process Group is dragged onto the canvas, the DFM is prompted to name the Process Group. All Process
Groups within the same parent group must have unique names. The Process Group will then be nested within that parent group.
+Once a Process Group has been dragged onto the canvas, the user may interact with it by right-clicking on the Process Group and selecting an option from
+context menu.
+
+image::nifi-process-group-menu.png["Process Group Menu", width=300]
+
+The following options are available:
+
+- *Configure*: This option allows the user to establish or change the configuration of the Process Group.
+- *Enter group*: This option allows the user to enter the Process Group. It is also possible to double-click on the Process Group to enter it.
+- *Start*: This option allows the user to start a Process Group.
+- *Stop*: This option allows the user to stop a Process Group.
+- *Stats*: This option opens a graphical representation of the Process Group's statistical information over time.
+- *Upstream connections*: This option allows the user to see and "jump to" upstream connections that are coming into the Process Group.
+- *Downstream connections*: This option allows the user to see and "jump to" downstream connections that are going out of the Process Group.
+- *Center in view*: This option centers the view of the canvas on the given Process Group.
+- *Copy* or *Paste*: This option places a copy of the selected Process Group on the clipboard, so that it may be pasted elsewhere on the canvas.
+- *Delete*: This option allows the DFM to delete a Process Group.
+
[[remote_process_group]]
@@ -241,6 +282,27 @@ how busy each node is. This information is then used to load balance the data th
then interrogated periodically to determine information about any nodes that are dropped from or added to the cluster and to
recalculate the load balancing based on each node's load. For more information, see the section on <<site-to-site,Site-to-Site>>.
+Once a Remote Process Group has been dragged onto the canvas, the user may interact with it by right-clicking on the Remote Process Group and selecting an option from
+context menu.
+
+image::nifi-rpg-menu.png["Remote Process Group Menu", width=300]
+
+The following options are available:
+
+- *Configure*: This option allows the user to establish or change the configuration of the Remote Process Group.
+- *Remote Ports*: This option allows the user to see input ports and/or output ports that exist on the remote instance of NiFi that the Remote Process Group is connected to. Note that if the site-to-site configuration is secure, only the ports that the connecting NiFi has been given accessed to will be visible.
+- *Enable transmission*: Makes the transmission of data between NiFi instances active. (See <<Remote_Group_Transmission>> )
+- *Disable transmission*: Disables the transmission of data between NiFi instances.
+- *Stats*: This option opens a graphical representation of the Remote Process Group's statistical information over time.
+- *Upstream connections*: This option allows the user to see and "jump to" upstream connections that are coming into the Remote Process Group.
+- *Downstream connections*: This option allows the user to see and "jump to" downstream connections that are going out of the Remote Process Group.
+- *Refresh*: This option refreshes the view of the status of the remote NiFi instance.
+- *Go to*: This option opens a view of the remote NiFi instance in a new tab of the browser. Note that if the site-to-site configuration is secure, the user must have access to the remote NiFi instance in order to view it.
+- *Center in view*: This option centers the view of the canvas on the given Remote Process Group.
+- *Copy* or *Paste*: This option places a copy of the selected Process Group on the clipboard, so that it may be pasted elsewhere on the canvas.
+- *Delete*: This option allows the DFM to delete a Remote Process Group from the canvas.
+
+
[[funnel]]
image:iconFunnel.png["Funnel", width=32]
@@ -276,11 +338,10 @@ choosing `Configure...`
-
+[[Configuring_a_Processor]]
=== Configuring a Processor
-Once a Processor has been dragged onto the Canvas, it is ready to configure. This is done by right-clicking on the
-Processor and clicking the `Configure...` option from the context menu. The configuration dialog is opened with four
+To configure a processor, right-click on the Processor and select the `Configure...` option from the context menu. The configuration dialog is opened with four
different tabs, each of which is discussed below. Once you have finished configuring the Processor, you can apply
the changes by clicking the `Apply` button or cancel all changes by clicking the `Cancel` button.
@@ -460,8 +521,12 @@ for all the Processors that are available. Clicking on the desired Processor in
While DFMs have the ability to create Controller Services from the Configure Processor window, there is also a central place within the User Interface for adding and configuring both Controller Services and Reporting Tasks. To get there, click on the Controller Settings button in the Management section of the toolbar.
+[[Controller_Settings]]
+==== Controller Settings
+
image:controller-settings-button.png["Controller Settings Button", width=200]
+
The Controller Settings window has three tabs across the top: General, Controller Services, and Reporting Tasks. The General tab is for settings that pertain to general information about the NiFi instance. For example, here, the DFM can provide a unique name for the overall dataflow, as well as comments that describe the flow. Be aware that this information is visible to any other NiFi instance that connects remotely to this instance (using Remote Process Groups, a.k.a., Site-to-Site).
The General tab also provides settings for the overall maximum thread counts of the instance, as well as the ability to click "Back-up flow" to create a backup copy of the current flow, which is saved by default in the /conf/archive directory.
@@ -472,7 +537,7 @@ To the right of the General tab is the Controller Services tab. From this tab, t
image:controller-services-tab.png["Controller Services Tab", width=900]
-The Add Controller Service window opens. This window is similar to the Add Processor window. It provides a list of the available Controller Services on the right and a tag cloud, showing the most common catagory tags used for Controller Services, on the left. The DFM may click any tag in the tag cloud in order to narrow down the list of Controller Services to those that fit the categories desired. The DFM may also use the Filter field at the top of the window to search for the desired Contoller Service. Upon selecting a Controller Service from the list, the DFM can see a description of the the service below. Select the desired controller service and click Add, or simply double-click the name of the service to add it.
+The Add Controller Service window opens. This window is similar to the Add Processor window. It provides a list of the available Controller Services on the right and a tag cloud, showing the most common category tags used for Controller Services, on the left. The DFM may click any tag in the tag cloud in order to narrow down the list of Controller Services to those that fit the categories desired. The DFM may also use the Filter field at the top of the window to search for the desired Controller Service. Upon selecting a Controller Service from the list, the DFM can see a description of the the service below. Select the desired controller service and click Add, or simply double-click the name of the service to add it.
image:add-controller-service-window.png["Add Controller Service Window", width=700]
@@ -513,7 +578,7 @@ The Comments tab is just an open-text field, where the DFM may include comments
When you want to run the Reporting Task, click the Start button in the far-right column of the Reporting Tasks tab.
-
+[[Connecting_Components]]
=== Connecting Components
Once processors and other components have been added to the graph and configured, the next step is to connect them
@@ -581,10 +646,28 @@ The following prioritizers are available:
- *FirstInFirstOutPrioritizer*: Given two FlowFiles, the on that reached the connection first will be processed first.
- *NewestFlowFileFirstPrioritizer*: Given two FlowFiles, the one that is newest in the dataflow will be processed first.
- *OldestFlowFileFirstPrioritizer*: Given two FlowFiles, the on that is oldest in the dataflow will be processed first. This is the default scheme that is used if no prioritizers are selected.
-- *PriorityAttributePrioritizer*: Given two FlowFiles that both have a "priority" attribute, the one that has the highest priority value will be prprocessed first. Note that an UpdateAttribute processor should be used to add the "priority" attribute to the FlowFiles before they reach a connection that has this prioritizer set. Values for the "priority" attribute may be alphanumeric, where "a" is a higher priority than "z", and "1" is a higher priority than "9", for example.
+- *PriorityAttributePrioritizer*: Given two FlowFiles that both have a "priority" attribute, the one that has the highest priority value will be processed first. Note that an UpdateAttribute processor should be used to add the "priority" attribute to the FlowFiles before they reach a connection that has this prioritizer set. Values for the "priority" attribute may be alphanumeric, where "a" is a higher priority than "z", and "1" is a higher priority than "9", for example.
*Note*: After a connection has been drawn between two components, the connection's configuration may be changed, and the connection may be moved to a new destination; however, the processors on either side of the connection must be stopped before a configuration or destination change may be made.
+image:nifi-connection.png["Connection", width=300]
+
+
+To change a connection's configuration or interact with the connection in other ways, right-click on the connection to open the connection context menu.
+
+image:nifi-connection-menu.png["Connection Menu", width=200]
+
+The following options are available:
+
+- *Configure*: This option allows the user to change the configuration of the connection.
+- *Stats*: This option opens a graphical representation of the connection's statistical information over time.
+- *Bring to front*: This option brings the connection to the front of the canvas if something else (such as another connection) is overlapping it.
+- *Go to source*: This option can be useful if there is a long distance between the connection's source and destination components on the canvas. By clicking this option, the view of the canvas will jump to the source of the connection.
+- *Go to destination*: Similar to the "Go to source" option, this option changes the view to the destination component on the canvas and can be useful if there is a long distance between two connected components.
+- *Empty queue*: This option allows the DFM to clear the queue of FlowFiles that may be waiting to be processed. This option can be especially useful during testing, when the DFM is not concerned about deleting data from the queue. When this option is selected, users must confirm that they want to delete the data in the queue.
+- *Delete*: This option allows the DFM to delete a connection between two components. Note that the components on both sides of the connection must be stopped and the connection must be empty before it can be deleted.
+
+
=== Processor Validation
Before trying to start a Processor, it's important to make sure that the Processor's configuration is valid.
@@ -717,7 +800,7 @@ Now see the following section on how to start and stop the dataflow. When the da
-== Command and Control of DataFlow
+== Command and Control of the DataFlow
When a component is added to the NiFi canvas, it is in the Stopped state. In order to cause the component to
be triggered, the component must be started. Once started, the component can be stopped at any time. From a
@@ -802,7 +885,7 @@ Only Ports and Processors can be enabled and disabled.
=== Remote Process Group Transmission
Remote Process Groups provide a mechanism for sending data to or retrieving data from a remote instance
-of NiFi. When a Remote Process Group (RPG) is added to the canvas, it is added with the Transmision Disabled,
+of NiFi. When a Remote Process Group (RPG) is added to the canvas, it is added with the Transmission Disabled,
as indicated by the icon (
image:iconTransmissionInactive.png["Transmission Disabled"]
) in the top-left corner. When Transmission is Disabled, it can be enabled by right-clicking on the
@@ -1118,7 +1201,7 @@ image:iconNotSecure.png["Not Secure"]
will be counted.
** image:iconTransmissionActive.png["Transmitting"]
- *Transmitting Ports*: The number of Output Ports from whcih this NiFi is connected and currently configured
+ *Transmitting Ports*: The number of Output Ports from which this NiFi is connected and currently configured
to pull data from. Ports can be turned on and off by enabling and disabling transmission on the Remote Process
Group (see <<Remote_Group_Transmission>>) or via the <<Remote_Port_Configuration>> dialog.
[04/50] [abbrv] nifi git commit: NIFI-1163: Providing handling of
SSLContext creation in GetHTTP in case of only performing a one-way SSL
request and accompanying test to verify the configuration/usage.
Posted by mc...@apache.org.
NIFI-1163: Providing handling of SSLContext creation in GetHTTP in case of only performing a one-way SSL request and accompanying test to verify the configuration/usage.
Reviewed by Tony Kurc (tkurc@apache.org)
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/01539ed3
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/01539ed3
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/01539ed3
Branch: refs/heads/NIFI-655
Commit: 01539ed3230894b062a7c2e42ffd9b50e3d51bf3
Parents: 90f6830
Author: Aldrin Piri <al...@apache.org>
Authored: Sat Nov 14 18:43:49 2015 -0500
Committer: Tony Kurc <tr...@gmail.com>
Committed: Sat Nov 14 18:47:50 2015 -0500
----------------------------------------------------------------------
.../nifi/processors/standard/GetHTTP.java | 28 +++---
.../nifi/processors/standard/TestGetHTTP.java | 94 ++++++++++++++------
2 files changed, 87 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/01539ed3/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetHTTP.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetHTTP.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetHTTP.java
index e846b82..2245080 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetHTTP.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetHTTP.java
@@ -49,6 +49,7 @@ import java.util.regex.Pattern;
import javax.net.ssl.SSLContext;
+import org.apache.commons.lang3.StringUtils;
import org.apache.http.Header;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
@@ -64,11 +65,11 @@ import org.apache.http.conn.HttpClientConnectionManager;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
-import org.apache.http.conn.ssl.SSLContexts;
import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
+import org.apache.http.ssl.SSLContextBuilder;
import org.apache.nifi.annotation.behavior.InputRequirement;
import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
import org.apache.nifi.annotation.behavior.WritesAttribute;
@@ -320,19 +321,26 @@ public class GetHTTP extends AbstractSessionFactoryProcessor {
private SSLContext createSSLContext(final SSLContextService service)
throws KeyStoreException, IOException, NoSuchAlgorithmException, CertificateException, KeyManagementException, UnrecoverableKeyException {
- final KeyStore truststore = KeyStore.getInstance(service.getTrustStoreType());
- try (final InputStream in = new FileInputStream(new File(service.getTrustStoreFile()))) {
- truststore.load(in, service.getTrustStorePassword().toCharArray());
- }
- final KeyStore keystore = KeyStore.getInstance(service.getKeyStoreType());
- try (final InputStream in = new FileInputStream(new File(service.getKeyStoreFile()))) {
- keystore.load(in, service.getKeyStorePassword().toCharArray());
+ final SSLContextBuilder sslContextBuilder = new SSLContextBuilder();
+
+ if (StringUtils.isNotBlank(service.getTrustStoreFile())) {
+ final KeyStore truststore = KeyStore.getInstance(service.getTrustStoreType());
+ try (final InputStream in = new FileInputStream(new File(service.getTrustStoreFile()))) {
+ truststore.load(in, service.getTrustStorePassword().toCharArray());
+ }
+ sslContextBuilder.loadTrustMaterial(truststore, new TrustSelfSignedStrategy());
}
- final SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(truststore, new TrustSelfSignedStrategy()).loadKeyMaterial(keystore, service.getKeyStorePassword().toCharArray()).build();
+ if (StringUtils.isNotBlank(service.getKeyStoreFile())){
+ final KeyStore keystore = KeyStore.getInstance(service.getKeyStoreType());
+ try (final InputStream in = new FileInputStream(new File(service.getKeyStoreFile()))) {
+ keystore.load(in, service.getKeyStorePassword().toCharArray());
+ }
+ sslContextBuilder.loadKeyMaterial(keystore, service.getKeyStorePassword().toCharArray());
+ }
- return sslContext;
+ return sslContextBuilder.build();
}
@Override
http://git-wip-us.apache.org/repos/asf/nifi/blob/01539ed3/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGetHTTP.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGetHTTP.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGetHTTP.java
index bb3d286..29ce429 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGetHTTP.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGetHTTP.java
@@ -317,48 +317,63 @@ public class TestGetHTTP {
}
}
- private Map<String, String> getSslProperties() {
- Map<String, String> props = new HashMap<String, String>();
- props.put(StandardSSLContextService.KEYSTORE.getName(), "src/test/resources/localhost-ks.jks");
- props.put(StandardSSLContextService.KEYSTORE_PASSWORD.getName(), "localtest");
- props.put(StandardSSLContextService.KEYSTORE_TYPE.getName(), "JKS");
- props.put(StandardSSLContextService.TRUSTSTORE.getName(), "src/test/resources/localhost-ts.jks");
- props.put(StandardSSLContextService.TRUSTSTORE_PASSWORD.getName(), "localtest");
- props.put(StandardSSLContextService.TRUSTSTORE_TYPE.getName(), "JKS");
- return props;
- }
+ @Test
+ public final void testSecure_oneWaySsl() throws Exception {
+ // set up web service
+ final ServletHandler handler = new ServletHandler();
+ handler.addServletWithMapping(HelloWorldServlet.class, "/*");
+
+ // create the service, disabling the need for client auth
+ final Map<String, String> serverSslProperties = getKeystoreProperties();
+ serverSslProperties.put(TestServer.NEED_CLIENT_AUTH, Boolean.toString(false));
+ final TestServer server = new TestServer(serverSslProperties);
+ server.addHandler(handler);
- private void useSSLContextService() {
- final SSLContextService service = new StandardSSLContextService();
try {
- controller.addControllerService("ssl-service", service, getSslProperties());
- controller.enableControllerService(service);
- } catch (InitializationException ex) {
- ex.printStackTrace();
- Assert.fail("Could not create SSL Context Service");
- }
+ server.startServer();
- controller.setProperty(GetHTTP.SSL_CONTEXT_SERVICE, "ssl-service");
+ final String destination = server.getSecureUrl();
+
+ // set up NiFi mock controller
+ controller = TestRunners.newTestRunner(GetHTTP.class);
+ // Use context service with only a truststore
+ useSSLContextService(getTruststoreProperties());
+
+ controller.setProperty(GetHTTP.CONNECTION_TIMEOUT, "5 secs");
+ controller.setProperty(GetHTTP.URL, destination);
+ controller.setProperty(GetHTTP.FILENAME, "testFile");
+ controller.setProperty(GetHTTP.ACCEPT_CONTENT_TYPE, "application/json");
+
+ controller.run();
+ controller.assertAllFlowFilesTransferred(GetHTTP.REL_SUCCESS, 1);
+ final MockFlowFile mff = controller.getFlowFilesForRelationship(GetHTTP.REL_SUCCESS).get(0);
+ mff.assertContentEquals("Hello, World!");
+ } finally {
+ server.shutdownServer();
+ }
}
@Test
- public final void testSecure() throws Exception {
+ public final void testSecure_twoWaySsl() throws Exception {
// set up web service
- ServletHandler handler = new ServletHandler();
+ final ServletHandler handler = new ServletHandler();
handler.addServletWithMapping(HelloWorldServlet.class, "/*");
- // create the service
- TestServer server = new TestServer(getSslProperties());
+ // create the service, providing both truststore and keystore properties, requiring client auth (default)
+ final Map<String, String> twoWaySslProperties = getKeystoreProperties();
+ twoWaySslProperties.putAll(getTruststoreProperties());
+ final TestServer server = new TestServer(twoWaySslProperties);
server.addHandler(handler);
try {
server.startServer();
- String destination = server.getSecureUrl();
+ final String destination = server.getSecureUrl();
// set up NiFi mock controller
controller = TestRunners.newTestRunner(GetHTTP.class);
- useSSLContextService();
+ // Use context service with a keystore and a truststore
+ useSSLContextService(twoWaySslProperties);
controller.setProperty(GetHTTP.CONNECTION_TIMEOUT, "5 secs");
controller.setProperty(GetHTTP.URL, destination);
@@ -374,4 +389,33 @@ public class TestGetHTTP {
}
}
+ private static Map<String, String> getTruststoreProperties() {
+ final Map<String, String> props = new HashMap<>();
+ props.put(StandardSSLContextService.TRUSTSTORE.getName(), "src/test/resources/localhost-ts.jks");
+ props.put(StandardSSLContextService.TRUSTSTORE_PASSWORD.getName(), "localtest");
+ props.put(StandardSSLContextService.TRUSTSTORE_TYPE.getName(), "JKS");
+ return props;
+ }
+
+ private static Map<String, String> getKeystoreProperties() {
+ final Map<String, String> properties = new HashMap<>();
+ properties.put(StandardSSLContextService.KEYSTORE.getName(), "src/test/resources/localhost-ks.jks");
+ properties.put(StandardSSLContextService.KEYSTORE_PASSWORD.getName(), "localtest");
+ properties.put(StandardSSLContextService.KEYSTORE_TYPE.getName(), "JKS");
+ return properties;
+ }
+
+ private void useSSLContextService(final Map<String, String> sslProperties) {
+ final SSLContextService service = new StandardSSLContextService();
+ try {
+ controller.addControllerService("ssl-service", service, sslProperties);
+ controller.enableControllerService(service);
+ } catch (InitializationException ex) {
+ ex.printStackTrace();
+ Assert.fail("Could not create SSL Context Service");
+ }
+
+ controller.setProperty(GetHTTP.SSL_CONTEXT_SERVICE, "ssl-service");
+ }
+
}
[02/50] [abbrv] nifi git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/nifi
Posted by mc...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/nifi
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/90f68300
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/90f68300
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/90f68300
Branch: refs/heads/NIFI-655
Commit: 90f6830003b76204e25dec9fafec2488c8bda550
Parents: 22de23b 36d00a6
Author: Mark Payne <ma...@hotmail.com>
Authored: Fri Nov 13 10:47:05 2015 -0500
Committer: Mark Payne <ma...@hotmail.com>
Committed: Fri Nov 13 10:47:05 2015 -0500
----------------------------------------------------------------------
.../java/org/apache/nifi/flowfile/FlowFile.java | 5 +
nifi-assembly/NOTICE | 7 +
nifi-assembly/pom.xml | 5 +
.../nifi-azure-bundle/nifi-azure-nar/pom.xml | 37 +++
.../src/main/resources/META-INF/NOTICE | 34 +++
.../nifi-azure-processors/pom.xml | 60 +++++
.../azure/eventhub/GetAzureEventHub.java | 254 +++++++++++++++++++
.../azure/eventhub/PutAzureEventHub.java | 186 ++++++++++++++
.../org.apache.nifi.processor.Processor | 16 ++
nifi-nar-bundles/nifi-azure-bundle/pom.xml | 35 +++
.../couchbase/AbstractCouchbaseProcessor.java | 109 ++++----
.../couchbase/CouchbaseExceptionMappings.java | 2 +-
.../processors/couchbase/GetCouchbaseKey.java | 114 +++++----
.../processors/couchbase/PutCouchbaseKey.java | 76 +++---
.../couchbase/TestGetCouchbaseKey.java | 17 +-
.../nifi/controller/StandardFlowFileQueue.java | 192 +++++++-------
.../controller/TestStandardFlowFileQueue.java | 175 +++++++++++++
.../hadoop/AbstractHadoopProcessor.java | 2 +-
.../nifi/processors/hadoop/FetchHDFS.java | 3 +
.../apache/nifi/processors/hadoop/ListHDFS.java | 3 +
.../nifi/processors/standard/ExecuteSQL.java | 8 +-
.../nifi/processors/standard/PutSyslog.java | 1 +
.../src/test/java/TestDynamicEnvironment.java | 29 +++
.../src/test/java/TestIngestAndUpdate.java | 34 +++
.../src/test/java/TestSuccess.java | 24 ++
nifi-nar-bundles/pom.xml | 7 +-
pom.xml | 6 +
27 files changed, 1182 insertions(+), 259 deletions(-)
----------------------------------------------------------------------
[05/50] [abbrv] nifi git commit: NIFI-1073: Fixing coverity
discovered errors. Resource leaks, and statics
Posted by mc...@apache.org.
NIFI-1073: Fixing coverity discovered errors. Resource leaks, and statics
Reviewed by Bryan Bende (bbende@apache.org)
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/528dab78
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/528dab78
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/528dab78
Branch: refs/heads/NIFI-655
Commit: 528dab78d6dbc3bb4f61663f423b07284936ec40
Parents: 01539ed
Author: Tony Kurc <tr...@gmail.com>
Authored: Mon Oct 26 19:41:26 2015 -0400
Committer: Tony Kurc <tr...@gmail.com>
Committed: Sat Nov 14 18:56:43 2015 -0500
----------------------------------------------------------------------
.../nifi/io/socket/SSLContextFactory.java | 15 ++-
.../stream/io/TestLeakyBucketThrottler.java | 102 +++++++++----------
.../nifi/processors/WriteResourceToStream.java | 10 +-
.../cluster/manager/impl/WebClusterManager.java | 12 +--
.../repository/FileSystemRepository.java | 9 +-
.../repository/StandardProcessSession.java | 8 +-
.../repository/VolatileContentRepository.java | 15 ++-
.../org/apache/nifi/web/server/JettyServer.java | 71 ++++++-------
.../org/apache/nifi/web/api/dto/DtoFactory.java | 2 +-
.../hadoop/TestCreateHadoopSequenceFile.java | 80 ++++++++-------
.../processors/standard/AttributesToJSON.java | 2 +-
.../processors/standard/ConvertJSONToSQL.java | 30 +++---
.../nifi/processors/standard/EncodeContent.java | 12 +--
.../nifi/processors/standard/GetHTTP.java | 5 +-
.../processors/standard/ListFileTransfer.java | 9 +-
.../nifi/processors/standard/PostHTTP.java | 2 +
.../nifi/processors/standard/SplitText.java | 2 +-
.../standard/util/OpenPGPKeyBasedEncryptor.java | 4 +-
.../util/OpenPGPPasswordBasedEncryptor.java | 4 +-
.../processors/standard/CaptureServlet.java | 12 ++-
.../nifi/processors/standard/TestGetHTTP.java | 2 +-
.../processors/standard/TestTransformXml.java | 31 +++---
.../pom.xml | 4 +
.../DistributedMapCacheClientService.java | 22 ++--
.../DistributedSetCacheClientService.java | 28 ++---
25 files changed, 271 insertions(+), 222 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/528dab78/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/SSLContextFactory.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/SSLContextFactory.java b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/SSLContextFactory.java
index 9c6cb82..ee75500 100644
--- a/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/SSLContextFactory.java
+++ b/nifi-commons/nifi-socket-utils/src/main/java/org/apache/nifi/io/socket/SSLContextFactory.java
@@ -34,6 +34,7 @@ import javax.net.ssl.TrustManager;
import javax.net.ssl.TrustManagerFactory;
import org.apache.nifi.util.NiFiProperties;
+import org.apache.nifi.util.file.FileUtils;
public class SSLContextFactory {
@@ -58,13 +59,23 @@ public class SSLContextFactory {
// prepare the keystore
final KeyStore keyStore = KeyStore.getInstance(keystoreType);
- keyStore.load(new FileInputStream(keystore), keystorePass);
+ final FileInputStream keyStoreStream = new FileInputStream(keystore);
+ try {
+ keyStore.load(keyStoreStream, keystorePass);
+ } finally {
+ FileUtils.closeQuietly(keyStoreStream);
+ }
final KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
keyManagerFactory.init(keyStore, keystorePass);
// prepare the truststore
final KeyStore trustStore = KeyStore.getInstance(truststoreType);
- trustStore.load(new FileInputStream(truststore), truststorePass);
+ final FileInputStream trustStoreStream = new FileInputStream(truststore);
+ try {
+ trustStore.load(trustStoreStream, truststorePass);
+ } finally {
+ FileUtils.closeQuietly(trustStoreStream);
+ }
final TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
trustManagerFactory.init(trustStore);
http://git-wip-us.apache.org/repos/asf/nifi/blob/528dab78/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/stream/io/TestLeakyBucketThrottler.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/stream/io/TestLeakyBucketThrottler.java b/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/stream/io/TestLeakyBucketThrottler.java
index 52bd8de..e46577a 100644
--- a/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/stream/io/TestLeakyBucketThrottler.java
+++ b/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/stream/io/TestLeakyBucketThrottler.java
@@ -41,75 +41,75 @@ public class TestLeakyBucketThrottler {
final byte[] data = new byte[1024 * 1024 * 4];
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
- final OutputStream throttledOut = throttler.newThrottledOutputStream(baos);
-
- final long start = System.currentTimeMillis();
- throttledOut.write(data);
- throttler.close();
- final long millis = System.currentTimeMillis() - start;
- // should take 4 sec give or take
- assertTrue(millis > 3000);
- assertTrue(millis < 6000);
+ try (final OutputStream throttledOut = throttler.newThrottledOutputStream(baos)) {
+
+ final long start = System.currentTimeMillis();
+ throttledOut.write(data);
+ throttler.close();
+ final long millis = System.currentTimeMillis() - start;
+ // should take 4 sec give or take
+ assertTrue(millis > 3000);
+ assertTrue(millis < 6000);
+ }
}
@Test(timeout = 10000)
public void testInputStreamInterface() throws IOException {
- // throttle rate at 1 MB/sec
- final LeakyBucketStreamThrottler throttler = new LeakyBucketStreamThrottler(1024 * 1024);
final byte[] data = new byte[1024 * 1024 * 4];
- final ByteArrayInputStream bais = new ByteArrayInputStream(data);
- final InputStream throttledIn = throttler.newThrottledInputStream(bais);
-
- final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ // throttle rate at 1 MB/sec
+ try ( final LeakyBucketStreamThrottler throttler = new LeakyBucketStreamThrottler(1024 * 1024);
+ final ByteArrayInputStream bais = new ByteArrayInputStream(data);
+ final InputStream throttledIn = throttler.newThrottledInputStream(bais);
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
+
+ final byte[] buffer = new byte[4096];
+ final long start = System.currentTimeMillis();
+ int len;
+ while ((len = throttledIn.read(buffer)) > 0) {
+ baos.write(buffer, 0, len);
+ }
- final byte[] buffer = new byte[4096];
- final long start = System.currentTimeMillis();
- int len;
- while ((len = throttledIn.read(buffer)) > 0) {
- baos.write(buffer, 0, len);
+ final long millis = System.currentTimeMillis() - start;
+ // should take 4 sec give or take
+ assertTrue(millis > 3000);
+ assertTrue(millis < 6000);
}
- throttler.close();
- final long millis = System.currentTimeMillis() - start;
- // should take 4 sec give or take
- assertTrue(millis > 3000);
- assertTrue(millis < 6000);
- baos.close();
}
@Test(timeout = 10000)
public void testDirectInterface() throws IOException, InterruptedException {
// throttle rate at 1 MB/sec
- final LeakyBucketStreamThrottler throttler = new LeakyBucketStreamThrottler(1024 * 1024);
-
- // create 3 threads, each sending ~2 MB
- final ByteArrayOutputStream baos = new ByteArrayOutputStream();
- final List<Thread> threads = new ArrayList<Thread>();
- for (int i = 0; i < 3; i++) {
- final Thread t = new WriterThread(i, throttler, baos);
- threads.add(t);
- }
+ try (final LeakyBucketStreamThrottler throttler = new LeakyBucketStreamThrottler(1024 * 1024);
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
+ // create 3 threads, each sending ~2 MB
+ final List<Thread> threads = new ArrayList<Thread>();
+ for (int i = 0; i < 3; i++) {
+ final Thread t = new WriterThread(i, throttler, baos);
+ threads.add(t);
+ }
- final long start = System.currentTimeMillis();
- for (final Thread t : threads) {
- t.start();
- }
+ final long start = System.currentTimeMillis();
+ for (final Thread t : threads) {
+ t.start();
+ }
- for (final Thread t : threads) {
- t.join();
- }
- final long elapsed = System.currentTimeMillis() - start;
+ for (final Thread t : threads) {
+ t.join();
+ }
+ final long elapsed = System.currentTimeMillis() - start;
- throttler.close();
+ throttler.close();
- // To send 15 MB, it should have taken at least 5 seconds and no more than 7 seconds, to
- // allow for busy-ness and the fact that we could write a tiny bit more than the limit.
- assertTrue(elapsed > 5000);
- assertTrue(elapsed < 7000);
+ // To send 15 MB, it should have taken at least 5 seconds and no more than 7 seconds, to
+ // allow for busy-ness and the fact that we could write a tiny bit more than the limit.
+ assertTrue(elapsed > 5000);
+ assertTrue(elapsed < 7000);
- // ensure bytes were copied out appropriately
- assertEquals(3 * (2 * 1024 * 1024 + 1), baos.getBufferLength());
- assertEquals((byte) 'A', baos.getUnderlyingBuffer()[baos.getBufferLength() - 1]);
+ // ensure bytes were copied out appropriately
+ assertEquals(3 * (2 * 1024 * 1024 + 1), baos.getBufferLength());
+ assertEquals((byte) 'A', baos.getUnderlyingBuffer()[baos.getBufferLength() - 1]);
+ }
}
private static class WriterThread extends Thread {
http://git-wip-us.apache.org/repos/asf/nifi/blob/528dab78/nifi-external/nifi-example-bundle/nifi-nifi-example-processors/src/main/java/org/apache/nifi/processors/WriteResourceToStream.java
----------------------------------------------------------------------
diff --git a/nifi-external/nifi-example-bundle/nifi-nifi-example-processors/src/main/java/org/apache/nifi/processors/WriteResourceToStream.java b/nifi-external/nifi-example-bundle/nifi-nifi-example-processors/src/main/java/org/apache/nifi/processors/WriteResourceToStream.java
index 5d595b4..55eb2d7 100644
--- a/nifi-external/nifi-example-bundle/nifi-nifi-example-processors/src/main/java/org/apache/nifi/processors/WriteResourceToStream.java
+++ b/nifi-external/nifi-example-bundle/nifi-nifi-example-processors/src/main/java/org/apache/nifi/processors/WriteResourceToStream.java
@@ -17,6 +17,7 @@
package org.apache.nifi.processors;
import java.io.IOException;
+import java.io.InputStream;
import java.io.OutputStream;
import java.util.Collections;
import java.util.HashSet;
@@ -57,13 +58,16 @@ public class WriteResourceToStream extends AbstractProcessor {
relationships.add(REL_SUCCESS);
relationships.add(REL_FAILURE);
this.relationships = Collections.unmodifiableSet(relationships);
-
+ final InputStream resourceStream = Thread.currentThread()
+ .getContextClassLoader().getResourceAsStream("file.txt");
try {
- this.resourceData = IOUtils.toString(Thread.currentThread()
- .getContextClassLoader().getResourceAsStream("file.txt"));
+ this.resourceData = IOUtils.toString(resourceStream);
} catch (IOException e) {
throw new RuntimeException("Unable to load resources", e);
+ } finally {
+ IOUtils.closeQuietly(resourceStream);
}
+
}
@Override
http://git-wip-us.apache.org/repos/asf/nifi/blob/528dab78/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/impl/WebClusterManager.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/impl/WebClusterManager.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/impl/WebClusterManager.java
index c22e1ec..5e9dd3c 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/impl/WebClusterManager.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/impl/WebClusterManager.java
@@ -3402,12 +3402,12 @@ public class WebClusterManager implements HttpClusterManager, ProtocolHandler, C
@Override
public void run() {
try {
- ((StreamingOutput) nodeResponse.getResponse().getEntity()).write(
- new OutputStream() {
- @Override
- public void write(final int b) { /* drain response */ }
- }
- );
+ try (final OutputStream drain = new OutputStream() {
+ @Override
+ public void write(final int b) { /* drain response */ }
+ }) {
+ ((StreamingOutput) nodeResponse.getResponse().getEntity()).write(drain);
+ }
} catch (final IOException | WebApplicationException ex) {
logger.info("Failed clearing out non-client response buffer due to: " + ex, ex);
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/528dab78/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java
index b4a1716..c72a19c 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java
@@ -58,6 +58,7 @@ import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.regex.Pattern;
+import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.controller.repository.claim.ContentClaim;
import org.apache.nifi.controller.repository.claim.ResourceClaim;
@@ -808,7 +809,13 @@ public class FileSystemRepository implements ContentRepository {
final Path path = getPath(claim, true);
final FileInputStream fis = new FileInputStream(path.toFile());
if (claim.getOffset() > 0L) {
- StreamUtils.skip(fis, claim.getOffset());
+ try {
+ StreamUtils.skip(fis, claim.getOffset());
+ } catch(IOException ioe) {
+ IOUtils.closeQuietly(fis);
+ throw ioe;
+ }
+
}
// see javadocs for claim.getLength() as to why we do this.
http://git-wip-us.apache.org/repos/asf/nifi/blob/528dab78/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
index f66894c..3709972 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
@@ -39,6 +39,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.regex.Pattern;
+import org.apache.commons.io.IOUtils;
import org.apache.nifi.connectable.Connectable;
import org.apache.nifi.connectable.Connection;
import org.apache.nifi.controller.ProcessorNode;
@@ -1759,7 +1760,12 @@ public final class StandardProcessSession implements ProcessSession, ProvenanceE
return new DisableOnCloseInputStream(currentReadClaimStream);
} else {
final InputStream rawInStream = context.getContentRepository().read(claim);
- StreamUtils.skip(rawInStream, offset);
+ try {
+ StreamUtils.skip(rawInStream, offset);
+ } catch(IOException ioe) {
+ IOUtils.closeQuietly(rawInStream);
+ throw ioe;
+ }
return rawInStream;
}
} catch (final ContentNotFoundException cnfe) {
http://git-wip-us.apache.org/repos/asf/nifi/blob/528dab78/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/VolatileContentRepository.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/VolatileContentRepository.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/VolatileContentRepository.java
index 7c7cade..08b7e80 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/VolatileContentRepository.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/VolatileContentRepository.java
@@ -39,6 +39,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
+import org.apache.commons.io.IOUtils;
import org.apache.nifi.controller.repository.claim.ContentClaim;
import org.apache.nifi.controller.repository.claim.ResourceClaim;
import org.apache.nifi.controller.repository.claim.ResourceClaimManager;
@@ -401,14 +402,22 @@ public class VolatileContentRepository implements ContentRepository {
@Override
public long exportTo(ContentClaim claim, OutputStream destination) throws IOException {
final InputStream in = read(claim);
- return StreamUtils.copy(in, destination);
+ try {
+ return StreamUtils.copy(in, destination);
+ } finally {
+ IOUtils.closeQuietly(in);
+ }
}
@Override
public long exportTo(ContentClaim claim, OutputStream destination, long offset, long length) throws IOException {
final InputStream in = read(claim);
- StreamUtils.skip(in, offset);
- StreamUtils.copy(in, destination, length);
+ try {
+ StreamUtils.skip(in, offset);
+ StreamUtils.copy(in, destination, length);
+ } finally {
+ IOUtils.closeQuietly(in);
+ }
return length;
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/528dab78/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/JettyServer.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/JettyServer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/JettyServer.java
index 99c11a8..ecfe2c0 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/JettyServer.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/JettyServer.java
@@ -38,8 +38,10 @@ import java.util.Map;
import java.util.Set;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
+
import javax.servlet.DispatcherType;
import javax.servlet.ServletContext;
+
import org.apache.nifi.NiFiServer;
import org.apache.nifi.controller.FlowSerializationException;
import org.apache.nifi.controller.FlowSynchronizationException;
@@ -51,6 +53,7 @@ import org.apache.nifi.services.FlowService;
import org.apache.nifi.util.NiFiProperties;
import org.apache.nifi.web.NiFiWebContext;
import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.web.ContentAccess;
import org.apache.nifi.ui.extension.UiExtension;
@@ -372,24 +375,25 @@ public class JettyServer implements NiFiServer {
}
// get an input stream for the nifi-processor configuration file
- BufferedReader in = new BufferedReader(new InputStreamReader(jarFile.getInputStream(jarEntry)));
-
- // read in each configured type
- String rawComponentType;
- while ((rawComponentType = in.readLine()) != null) {
- // extract the component type
- final String componentType = extractComponentType(rawComponentType);
- if (componentType != null) {
- List<String> extensions = uiExtensions.get(uiExtensionType);
-
- // if there are currently no extensions for this type create it
- if (extensions == null) {
- extensions = new ArrayList<>();
- uiExtensions.put(uiExtensionType, extensions);
- }
+ try (BufferedReader in = new BufferedReader(new InputStreamReader(jarFile.getInputStream(jarEntry)))) {
+
+ // read in each configured type
+ String rawComponentType;
+ while ((rawComponentType = in.readLine()) != null) {
+ // extract the component type
+ final String componentType = extractComponentType(rawComponentType);
+ if (componentType != null) {
+ List<String> extensions = uiExtensions.get(uiExtensionType);
+
+ // if there are currently no extensions for this type create it
+ if (extensions == null) {
+ extensions = new ArrayList<>();
+ uiExtensions.put(uiExtensionType, extensions);
+ }
- // add the specified type
- extensions.add(componentType);
+ // add the specified type
+ extensions.add(componentType);
+ }
}
}
}
@@ -437,37 +441,34 @@ public class JettyServer implements NiFiServer {
*/
private List<String> getWarExtensions(final File war, final String path) {
List<String> processorTypes = new ArrayList<>();
+
+ // load the jar file and attempt to find the nifi-processor entry
JarFile jarFile = null;
try {
- // load the jar file and attempt to find the nifi-processor entry
jarFile = new JarFile(war);
JarEntry jarEntry = jarFile.getJarEntry(path);
// ensure the nifi-processor entry was found
if (jarEntry != null) {
// get an input stream for the nifi-processor configuration file
- BufferedReader in = new BufferedReader(new InputStreamReader(jarFile.getInputStream(jarEntry)));
-
- // read in each configured type
- String rawProcessorType;
- while ((rawProcessorType = in.readLine()) != null) {
- // extract the processor type
- final String processorType = extractComponentType(rawProcessorType);
- if (processorType != null) {
- processorTypes.add(processorType);
+ try (final BufferedReader in = new BufferedReader(
+ new InputStreamReader(jarFile.getInputStream(jarEntry)))) {
+
+ // read in each configured type
+ String rawProcessorType;
+ while ((rawProcessorType = in.readLine()) != null) {
+ // extract the processor type
+ final String processorType = extractComponentType(rawProcessorType);
+ if (processorType != null) {
+ processorTypes.add(processorType);
+ }
}
}
}
} catch (IOException ioe) {
- logger.warn(String.format("Unable to inspect %s for a custom processor UI.", war));
+ logger.warn("Unable to inspect {} for a custom processor UI.", new Object[]{war, ioe});
} finally {
- try {
- // close the jar file - which closes all input streams obtained via getInputStream above
- if (jarFile != null) {
- jarFile.close();
- }
- } catch (IOException ioe) {
- }
+ IOUtils.closeQuietly(jarFile);
}
return processorTypes;
http://git-wip-us.apache.org/repos/asf/nifi/blob/528dab78/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
index c71c0cf..87277ae 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java
@@ -148,7 +148,7 @@ public final class DtoFactory {
}
};
- final int MAX_BULLETINS_PER_COMPONENT = 5;
+ final static int MAX_BULLETINS_PER_COMPONENT = 5;
private ControllerServiceLookup controllerServiceLookup;
http://git-wip-us.apache.org/repos/asf/nifi/blob/528dab78/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/TestCreateHadoopSequenceFile.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/TestCreateHadoopSequenceFile.java b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/TestCreateHadoopSequenceFile.java
index 430b4fa..5357dff 100644
--- a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/TestCreateHadoopSequenceFile.java
+++ b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/TestCreateHadoopSequenceFile.java
@@ -22,7 +22,6 @@ import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.FileInputStream;
-import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.HashMap;
@@ -71,9 +70,11 @@ public class TestCreateHadoopSequenceFile {
}
@Test
- public void testSimpleCase() throws FileNotFoundException {
+ public void testSimpleCase() throws IOException {
for (File inFile : inFiles) {
- controller.enqueue(new FileInputStream(inFile));
+ try (FileInputStream fin = new FileInputStream(inFile) ) {
+ controller.enqueue(fin);
+ }
}
controller.run(3);
@@ -88,7 +89,9 @@ public class TestCreateHadoopSequenceFile {
@Test
public void testSequenceFileSaysValueIsBytesWritable() throws UnsupportedEncodingException, IOException {
for (File inFile : inFiles) {
- controller.enqueue(new FileInputStream(inFile));
+ try (FileInputStream fin = new FileInputStream(inFile) ){
+ controller.enqueue(fin);
+ }
}
controller.run(3);
@@ -118,35 +121,39 @@ public class TestCreateHadoopSequenceFile {
}
@Test
- public void testMergedTarData() throws FileNotFoundException {
+ public void testMergedTarData() throws IOException {
Map<String, String> attributes = new HashMap<>();
attributes.put(CoreAttributes.MIME_TYPE.key(), "application/tar");
- controller.enqueue(new FileInputStream("src/test/resources/testdata/13545312236534130.tar"), attributes);
- controller.run();
- List<MockFlowFile> successSeqFiles = controller.getFlowFilesForRelationship(CreateHadoopSequenceFile.RELATIONSHIP_SUCCESS);
- assertEquals(1, successSeqFiles.size());
- final byte[] data = successSeqFiles.iterator().next().toByteArray();
- // Data should be greater than 1000000 because that's the size of 2 of our input files,
- // and the file size should contain all of that plus headers, but the headers should only
- // be a couple hundred bytes.
- assertTrue(data.length > 1000000);
- assertTrue(data.length < 1501000);
+ try (final FileInputStream fin = new FileInputStream("src/test/resources/testdata/13545312236534130.tar")) {
+ controller.enqueue(fin, attributes);
+ controller.run();
+ List<MockFlowFile> successSeqFiles = controller.getFlowFilesForRelationship(CreateHadoopSequenceFile.RELATIONSHIP_SUCCESS);
+ assertEquals(1, successSeqFiles.size());
+ final byte[] data = successSeqFiles.iterator().next().toByteArray();
+ // Data should be greater than 1000000 because that's the size of 2 of our input files,
+ // and the file size should contain all of that plus headers, but the headers should only
+ // be a couple hundred bytes.
+ assertTrue(data.length > 1000000);
+ assertTrue(data.length < 1501000);
+ }
}
@Test
public void testMergedZipData() throws IOException {
Map<String, String> attributes = new HashMap<>();
attributes.put(CoreAttributes.MIME_TYPE.key(), "application/zip");
- controller.enqueue(new FileInputStream("src/test/resources/testdata/13545423550275052.zip"), attributes);
- controller.run();
- List<MockFlowFile> successSeqFiles = controller.getFlowFilesForRelationship(CreateHadoopSequenceFile.RELATIONSHIP_SUCCESS);
- assertEquals(1, successSeqFiles.size());
- final byte[] data = successSeqFiles.iterator().next().toByteArray();
- // Data should be greater than 1000000 because that's the size of 2 of our input files,
- // and the file size should contain all of that plus headers, but the headers should only
- // be a couple hundred bytes.
- assertTrue(data.length > 1000000);
- assertTrue(data.length < 1501000);
+ try (FileInputStream fin = new FileInputStream("src/test/resources/testdata/13545423550275052.zip")){
+ controller.enqueue(fin, attributes);
+ controller.run();
+ List<MockFlowFile> successSeqFiles = controller.getFlowFilesForRelationship(CreateHadoopSequenceFile.RELATIONSHIP_SUCCESS);
+ assertEquals(1, successSeqFiles.size());
+ final byte[] data = successSeqFiles.iterator().next().toByteArray();
+ // Data should be greater than 1000000 because that's the size of 2 of our input files,
+ // and the file size should contain all of that plus headers, but the headers should only
+ // be a couple hundred bytes.
+ assertTrue(data.length > 1000000);
+ assertTrue(data.length < 1501000);
+ }
// FileOutputStream fos = new FileOutputStream("zip-3-randoms.sf");
// fos.write(data);
// fos.flush();
@@ -157,16 +164,19 @@ public class TestCreateHadoopSequenceFile {
public void testMergedFlowfilePackagedData() throws IOException {
Map<String, String> attributes = new HashMap<>();
attributes.put(CoreAttributes.MIME_TYPE.key(), "application/flowfile-v3");
- controller.enqueue(new FileInputStream("src/test/resources/testdata/13545479542069498.pkg"), attributes);
- controller.run();
- List<MockFlowFile> successSeqFiles = controller.getFlowFilesForRelationship(CreateHadoopSequenceFile.RELATIONSHIP_SUCCESS);
- assertEquals(1, successSeqFiles.size());
- final byte[] data = successSeqFiles.iterator().next().toByteArray();
- // Data should be greater than 1000000 because that's the size of 2 of our input files,
- // and the file size should contain all of that plus headers, but the headers should only
- // be a couple hundred bytes.
- assertTrue(data.length > 1000000);
- assertTrue(data.length < 1501000);
+ try ( final FileInputStream fin = new FileInputStream("src/test/resources/testdata/13545479542069498.pkg")) {
+ controller.enqueue(fin, attributes);
+
+ controller.run();
+ List<MockFlowFile> successSeqFiles = controller.getFlowFilesForRelationship(CreateHadoopSequenceFile.RELATIONSHIP_SUCCESS);
+ assertEquals(1, successSeqFiles.size());
+ final byte[] data = successSeqFiles.iterator().next().toByteArray();
+ // Data should be greater than 1000000 because that's the size of 2 of our input files,
+ // and the file size should contain all of that plus headers, but the headers should only
+ // be a couple hundred bytes.
+ assertTrue(data.length > 1000000);
+ assertTrue(data.length < 1501000);
+ }
// FileOutputStream fos = new FileOutputStream("flowfilePkg-3-randoms.sf");
// fos.write(data);
// fos.flush();
http://git-wip-us.apache.org/repos/asf/nifi/blob/528dab78/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AttributesToJSON.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AttributesToJSON.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AttributesToJSON.java
index 0ab3ca7..78b8d58 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AttributesToJSON.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AttributesToJSON.java
@@ -66,7 +66,7 @@ public class AttributesToJSON extends AbstractProcessor {
public static final String DESTINATION_ATTRIBUTE = "flowfile-attribute";
public static final String DESTINATION_CONTENT = "flowfile-content";
- private final String APPLICATION_JSON = "application/json";
+ private static final String APPLICATION_JSON = "application/json";
public static final PropertyDescriptor ATTRIBUTES_LIST = new PropertyDescriptor.Builder()
http://git-wip-us.apache.org/repos/asf/nifi/blob/528dab78/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java
index 9591960..6f14800 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java
@@ -561,25 +561,27 @@ public class ConvertJSONToSQL extends AbstractProcessor {
public static TableSchema from(final Connection conn, final String catalog, final String tableName,
final boolean translateColumnNames, final boolean includePrimaryKeys) throws SQLException {
- final ResultSet colrs = conn.getMetaData().getColumns(catalog, null, tableName, "%");
+ try (final ResultSet colrs = conn.getMetaData().getColumns(catalog, null, tableName, "%")) {
- final List<ColumnDescription> cols = new ArrayList<>();
- while (colrs.next()) {
- final ColumnDescription col = ColumnDescription.from(colrs);
- cols.add(col);
- }
+ final List<ColumnDescription> cols = new ArrayList<>();
+ while (colrs.next()) {
+ final ColumnDescription col = ColumnDescription.from(colrs);
+ cols.add(col);
+ }
- final Set<String> primaryKeyColumns = new HashSet<>();
- if (includePrimaryKeys) {
- final ResultSet pkrs = conn.getMetaData().getPrimaryKeys(catalog, null, tableName);
+ final Set<String> primaryKeyColumns = new HashSet<>();
+ if (includePrimaryKeys) {
+ try (final ResultSet pkrs = conn.getMetaData().getPrimaryKeys(catalog, null, tableName)) {
- while (pkrs.next()) {
- final String colName = pkrs.getString("COLUMN_NAME");
- primaryKeyColumns.add(normalizeColumnName(colName, translateColumnNames));
+ while (pkrs.next()) {
+ final String colName = pkrs.getString("COLUMN_NAME");
+ primaryKeyColumns.add(normalizeColumnName(colName, translateColumnNames));
+ }
+ }
}
- }
- return new TableSchema(cols, translateColumnNames, primaryKeyColumns);
+ return new TableSchema(cols, translateColumnNames, primaryKeyColumns);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/528dab78/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EncodeContent.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EncodeContent.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EncodeContent.java
index de81fe5..d1e010d 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EncodeContent.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EncodeContent.java
@@ -171,7 +171,7 @@ public class EncodeContent extends AbstractProcessor {
}
}
- private class EncodeBase64 implements StreamCallback {
+ private static class EncodeBase64 implements StreamCallback {
@Override
public void process(InputStream in, OutputStream out) throws IOException {
@@ -181,7 +181,7 @@ public class EncodeContent extends AbstractProcessor {
}
}
- private class DecodeBase64 implements StreamCallback {
+ private static class DecodeBase64 implements StreamCallback {
@Override
public void process(InputStream in, OutputStream out) throws IOException {
@@ -191,7 +191,7 @@ public class EncodeContent extends AbstractProcessor {
}
}
- private class EncodeBase32 implements StreamCallback {
+ private static class EncodeBase32 implements StreamCallback {
@Override
public void process(InputStream in, OutputStream out) throws IOException {
@@ -201,7 +201,7 @@ public class EncodeContent extends AbstractProcessor {
}
}
- private class DecodeBase32 implements StreamCallback {
+ private static class DecodeBase32 implements StreamCallback {
@Override
public void process(InputStream in, OutputStream out) throws IOException {
@@ -213,7 +213,7 @@ public class EncodeContent extends AbstractProcessor {
private static final byte[] HEX_CHARS = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'};
- private class EncodeHex implements StreamCallback {
+ private static class EncodeHex implements StreamCallback {
@Override
public void process(InputStream in, OutputStream out) throws IOException {
@@ -231,7 +231,7 @@ public class EncodeContent extends AbstractProcessor {
}
}
- private class DecodeHex implements StreamCallback {
+ private static class DecodeHex implements StreamCallback {
@Override
public void process(InputStream in, OutputStream out) throws IOException {
http://git-wip-us.apache.org/repos/asf/nifi/blob/528dab78/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetHTTP.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetHTTP.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetHTTP.java
index 2245080..5eab704 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetHTTP.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetHTTP.java
@@ -56,7 +56,6 @@ import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
-import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.config.Registry;
@@ -67,6 +66,7 @@ import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
import org.apache.http.ssl.SSLContextBuilder;
@@ -438,7 +438,7 @@ public class GetHTTP extends AbstractSessionFactoryProcessor {
}
// create the http client
- final HttpClient client = clientBuilder.build();
+ final CloseableHttpClient client = clientBuilder.build();
// create request
final HttpGet get = new HttpGet(url);
@@ -539,7 +539,6 @@ public class GetHTTP extends AbstractSessionFactoryProcessor {
logger.error("Failed to process due to {}; rolling back session", new Object[]{t.getMessage()}, t);
throw t;
}
-
} finally {
conMan.shutdown();
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/528dab78/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListFileTransfer.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListFileTransfer.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListFileTransfer.java
index b6c8c28..2cc3aae 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListFileTransfer.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListFileTransfer.java
@@ -23,6 +23,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import org.apache.commons.io.IOUtils;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.flowfile.attributes.CoreAttributes;
import org.apache.nifi.processor.ProcessContext;
@@ -92,7 +93,13 @@ public abstract class ListFileTransfer extends AbstractListProcessor<FileInfo> {
@Override
protected List<FileInfo> performListing(final ProcessContext context, final Long minTimestamp) throws IOException {
final FileTransfer transfer = getFileTransfer(context);
- final List<FileInfo> listing = transfer.getListing();
+ final List<FileInfo> listing;
+ try {
+ listing = transfer.getListing();
+ } finally {
+ IOUtils.closeQuietly(transfer);
+ }
+
if (minTimestamp == null) {
return listing;
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/528dab78/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PostHTTP.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PostHTTP.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PostHTTP.java
index ef84629..c830ac0 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PostHTTP.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PostHTTP.java
@@ -49,6 +49,7 @@ import javax.net.ssl.SSLSession;
import javax.security.cert.X509Certificate;
import javax.servlet.http.HttpServletResponse;
+import org.apache.commons.io.IOUtils;
import org.apache.http.Header;
import org.apache.http.HttpException;
import org.apache.http.HttpHost;
@@ -637,6 +638,7 @@ public class PostHTTP extends AbstractProcessor {
+ "configured to deliver FlowFiles; rolling back session", new Object[]{url});
session.rollback();
context.yield();
+ IOUtils.closeQuietly(client);
return;
}
} else {
http://git-wip-us.apache.org/repos/asf/nifi/blob/528dab78/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitText.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitText.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitText.java
index e966880..678919c 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitText.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitText.java
@@ -353,7 +353,7 @@ public class SplitText extends AbstractProcessor {
}
}
- private class SplitInfo {
+ private static class SplitInfo {
public long offsetBytes;
public long lengthBytes;
http://git-wip-us.apache.org/repos/asf/nifi/blob/528dab78/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/OpenPGPKeyBasedEncryptor.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/OpenPGPKeyBasedEncryptor.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/OpenPGPKeyBasedEncryptor.java
index f41d603..b91a529 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/OpenPGPKeyBasedEncryptor.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/OpenPGPKeyBasedEncryptor.java
@@ -134,7 +134,7 @@ public class OpenPGPKeyBasedEncryptor implements Encryptor {
return null;
}
- private class OpenPGPDecryptCallback implements StreamCallback {
+ private static class OpenPGPDecryptCallback implements StreamCallback {
private String provider;
private String secretKeyring;
@@ -216,7 +216,7 @@ public class OpenPGPKeyBasedEncryptor implements Encryptor {
}
- private class OpenPGPEncryptCallback implements StreamCallback {
+ private static class OpenPGPEncryptCallback implements StreamCallback {
private String algorithm;
private String provider;
http://git-wip-us.apache.org/repos/asf/nifi/blob/528dab78/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/OpenPGPPasswordBasedEncryptor.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/OpenPGPPasswordBasedEncryptor.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/OpenPGPPasswordBasedEncryptor.java
index 3e870df..b09d444 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/OpenPGPPasswordBasedEncryptor.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/OpenPGPPasswordBasedEncryptor.java
@@ -65,7 +65,7 @@ public class OpenPGPPasswordBasedEncryptor implements Encryptor {
return new OpenPGPDecryptCallback(provider, password);
}
- private class OpenPGPDecryptCallback implements StreamCallback {
+ private static class OpenPGPDecryptCallback implements StreamCallback {
private String provider;
private char[] password;
@@ -120,7 +120,7 @@ public class OpenPGPPasswordBasedEncryptor implements Encryptor {
}
- private class OpenPGPEncryptCallback implements StreamCallback {
+ private static class OpenPGPEncryptCallback implements StreamCallback {
private String algorithm;
private String provider;
http://git-wip-us.apache.org/repos/asf/nifi/blob/528dab78/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/CaptureServlet.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/CaptureServlet.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/CaptureServlet.java
index d6c87d6..a1398f4 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/CaptureServlet.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/CaptureServlet.java
@@ -24,8 +24,9 @@ import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.core.Response.Status;
-import org.apache.activemq.util.ByteArrayOutputStream;
+import org.apache.nifi.stream.io.ByteArrayOutputStream;
import org.apache.nifi.stream.io.StreamUtils;
+import org.apache.nifi.util.file.FileUtils;
public class CaptureServlet extends HttpServlet {
@@ -40,9 +41,12 @@ public class CaptureServlet extends HttpServlet {
@Override
protected void doPost(final HttpServletRequest request, final HttpServletResponse response) throws ServletException, IOException {
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
- StreamUtils.copy(request.getInputStream(), baos);
- this.lastPost = baos.toByteArray();
-
+ try{
+ StreamUtils.copy(request.getInputStream(), baos);
+ this.lastPost = baos.toByteArray();
+ } finally{
+ FileUtils.closeQuietly(baos);
+ }
response.setStatus(Status.OK.getStatusCode());
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/528dab78/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGetHTTP.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGetHTTP.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGetHTTP.java
index 29ce429..5aec796 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGetHTTP.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGetHTTP.java
@@ -375,7 +375,7 @@ public class TestGetHTTP {
// Use context service with a keystore and a truststore
useSSLContextService(twoWaySslProperties);
- controller.setProperty(GetHTTP.CONNECTION_TIMEOUT, "5 secs");
+ controller.setProperty(GetHTTP.CONNECTION_TIMEOUT, "10 secs");
controller.setProperty(GetHTTP.URL, destination);
controller.setProperty(GetHTTP.FILENAME, "testFile");
controller.setProperty(GetHTTP.ACCEPT_CONTENT_TYPE, "application/json");
http://git-wip-us.apache.org/repos/asf/nifi/blob/528dab78/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTransformXml.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTransformXml.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTransformXml.java
index 7074ec9..2dbf09f 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTransformXml.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTransformXml.java
@@ -20,7 +20,6 @@ import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
-import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Paths;
@@ -93,23 +92,25 @@ public class TestTransformXml {
StringBuilder builder = new StringBuilder();
builder.append("<data>\n");
- InputStream in = new FileInputStream(new File("src/test/resources/TestTransformXml/tokens.csv"));
- BufferedReader reader = new BufferedReader(new InputStreamReader(in));
+ try(BufferedReader reader = new BufferedReader(new InputStreamReader(
+ new FileInputStream(new File("src/test/resources/TestTransformXml/tokens.csv"))))){
- String line = null;
- while ((line = reader.readLine()) != null) {
- builder.append(line).append("\n");
- }
- builder.append("</data>");
- String data = builder.toString();
- runner.enqueue(data.getBytes(), attributes);
- runner.run();
- runner.assertAllFlowFilesTransferred(TransformXml.REL_SUCCESS);
- final MockFlowFile transformed = runner.getFlowFilesForRelationship(TransformXml.REL_SUCCESS).get(0);
- final String transformedContent = new String(transformed.toByteArray(), StandardCharsets.ISO_8859_1);
+ String line = null;
+ while ((line = reader.readLine()) != null) {
+ builder.append(line).append("\n");
+ }
+ builder.append("</data>");
+ String data = builder.toString();
+ runner.enqueue(data.getBytes(), attributes);
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred(TransformXml.REL_SUCCESS);
+ final MockFlowFile transformed = runner.getFlowFilesForRelationship(TransformXml.REL_SUCCESS).get(0);
+ final String transformedContent = new String(transformed.toByteArray(), StandardCharsets.ISO_8859_1);
- transformed.assertContentEquals(Paths.get("src/test/resources/TestTransformXml/tokens.xml"));
+ transformed.assertContentEquals(Paths.get("src/test/resources/TestTransformXml/tokens.xml"));
+ }
}
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/528dab78/nifi-nar-bundles/nifi-standard-services/nifi-distributed-cache-services-bundle/nifi-distributed-cache-client-service/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-distributed-cache-services-bundle/nifi-distributed-cache-client-service/pom.xml b/nifi-nar-bundles/nifi-standard-services/nifi-distributed-cache-services-bundle/nifi-distributed-cache-client-service/pom.xml
index e05f4dc..e335b1e 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-distributed-cache-services-bundle/nifi-distributed-cache-client-service/pom.xml
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-distributed-cache-services-bundle/nifi-distributed-cache-client-service/pom.xml
@@ -47,5 +47,9 @@
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-ssl-context-service-api</artifactId>
</dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/528dab78/nifi-nar-bundles/nifi-standard-services/nifi-distributed-cache-services-bundle/nifi-distributed-cache-client-service/src/main/java/org/apache/nifi/distributed/cache/client/DistributedMapCacheClientService.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-distributed-cache-services-bundle/nifi-distributed-cache-client-service/src/main/java/org/apache/nifi/distributed/cache/client/DistributedMapCacheClientService.java b/nifi-nar-bundles/nifi-standard-services/nifi-distributed-cache-services-bundle/nifi-distributed-cache-client-service/src/main/java/org/apache/nifi/distributed/cache/client/DistributedMapCacheClientService.java
index c03dd5a..e78ae0b 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-distributed-cache-services-bundle/nifi-distributed-cache-client-service/src/main/java/org/apache/nifi/distributed/cache/client/DistributedMapCacheClientService.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-distributed-cache-services-bundle/nifi-distributed-cache-client-service/src/main/java/org/apache/nifi/distributed/cache/client/DistributedMapCacheClientService.java
@@ -24,6 +24,7 @@ import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
+import org.apache.commons.io.IOUtils;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.SeeAlso;
import org.apache.nifi.annotation.documentation.Tags;
@@ -298,27 +299,18 @@ public class DistributedMapCacheClientService extends AbstractControllerService
if (closed) {
throw new IllegalStateException("Client is closed");
}
-
+ boolean tryToRequeue = true;
final CommsSession session = leaseCommsSession();
try {
return action.execute(session);
} catch (final IOException ioe) {
- try {
- session.close();
- } catch (final IOException ignored) {
- }
-
+ tryToRequeue = false;
throw ioe;
} finally {
- if (!session.isClosed()) {
- if (this.closed) {
- try {
- session.close();
- } catch (final IOException ioe) {
- }
- } else {
- queue.offer(session);
- }
+ if (tryToRequeue == true && this.closed == false) {
+ queue.offer(session);
+ } else {
+ IOUtils.closeQuietly(session);
}
}
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/528dab78/nifi-nar-bundles/nifi-standard-services/nifi-distributed-cache-services-bundle/nifi-distributed-cache-client-service/src/main/java/org/apache/nifi/distributed/cache/client/DistributedSetCacheClientService.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-distributed-cache-services-bundle/nifi-distributed-cache-client-service/src/main/java/org/apache/nifi/distributed/cache/client/DistributedSetCacheClientService.java b/nifi-nar-bundles/nifi-standard-services/nifi-distributed-cache-services-bundle/nifi-distributed-cache-client-service/src/main/java/org/apache/nifi/distributed/cache/client/DistributedSetCacheClientService.java
index 8c95c77..82ab643 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-distributed-cache-services-bundle/nifi-distributed-cache-client-service/src/main/java/org/apache/nifi/distributed/cache/client/DistributedSetCacheClientService.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-distributed-cache-services-bundle/nifi-distributed-cache-client-service/src/main/java/org/apache/nifi/distributed/cache/client/DistributedSetCacheClientService.java
@@ -24,6 +24,7 @@ import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
+import org.apache.commons.io.IOUtils;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.SeeAlso;
import org.apache.nifi.annotation.documentation.Tags;
@@ -127,10 +128,7 @@ public class DistributedSetCacheClientService extends AbstractControllerService
try {
ProtocolHandshake.initiateHandshake(session.getInputStream(), session.getOutputStream(), versionNegotiator);
} catch (final HandshakeException e) {
- try {
- session.close();
- } catch (final IOException ioe) {
- }
+ IOUtils.closeQuietly(session);
throw new IOException(e);
}
@@ -162,9 +160,9 @@ public class DistributedSetCacheClientService extends AbstractControllerService
try (final DataOutputStream dos = new DataOutputStream(commsSession.getOutputStream())) {
dos.writeUTF("close");
dos.flush();
- commsSession.close();
} catch (final IOException e) {
}
+ IOUtils.closeQuietly(commsSession);
}
if (logger.isDebugEnabled() && getIdentifier() != null) {
logger.debug("Closed {}", new Object[]{getIdentifier()});
@@ -185,6 +183,7 @@ public class DistributedSetCacheClientService extends AbstractControllerService
}
final CommsSession session = leaseCommsSession();
+ boolean tryToRequeue = true;
try {
final DataOutputStream dos = new DataOutputStream(session.getOutputStream());
dos.writeUTF(methodName);
@@ -198,22 +197,13 @@ public class DistributedSetCacheClientService extends AbstractControllerService
final DataInputStream dis = new DataInputStream(session.getInputStream());
return dis.readBoolean();
} catch (final IOException ioe) {
- try {
- session.close();
- } catch (final IOException ignored) {
- }
-
+ tryToRequeue = false;
throw ioe;
} finally {
- if (!session.isClosed()) {
- if (this.closed) {
- try {
- session.close();
- } catch (final IOException ioe) {
- }
- } else {
- queue.offer(session);
- }
+ if (tryToRequeue == true && this.closed == false) {
+ queue.offer(session);
+ } else {
+ IOUtils.closeQuietly(session);
}
}
}
[22/50] [abbrv] nifi git commit: NIFI-1187: Fixing issue of possible
assigment reordering causing uninitalized values to be possibly returned
Posted by mc...@apache.org.
NIFI-1187: Fixing issue of possible assigment reordering causing uninitalized values to be possibly returned
Signed-off-by: Aldrin Piri <al...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/ab794036
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/ab794036
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/ab794036
Branch: refs/heads/NIFI-655
Commit: ab7940368a18e8d29d8fadcc435db8ae8b4ee001
Parents: c541c82
Author: Tony Kurc <tr...@gmail.com>
Authored: Tue Nov 17 22:16:24 2015 -0500
Committer: Aldrin Piri <al...@apache.org>
Committed: Wed Nov 18 17:37:47 2015 -0500
----------------------------------------------------------------------
.../couchbase/AbstractCouchbaseProcessor.java | 8 +++-----
.../nifi/processors/flume/util/FlowFileEvent.java | 15 +++------------
2 files changed, 6 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/ab794036/nifi-nar-bundles/nifi-couchbase-bundle/nifi-couchbase-processors/src/main/java/org/apache/nifi/processors/couchbase/AbstractCouchbaseProcessor.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-couchbase-bundle/nifi-couchbase-processors/src/main/java/org/apache/nifi/processors/couchbase/AbstractCouchbaseProcessor.java b/nifi-nar-bundles/nifi-couchbase-bundle/nifi-couchbase-processors/src/main/java/org/apache/nifi/processors/couchbase/AbstractCouchbaseProcessor.java
index 158caa1..ec0e9ab 100644
--- a/nifi-nar-bundles/nifi-couchbase-bundle/nifi-couchbase-processors/src/main/java/org/apache/nifi/processors/couchbase/AbstractCouchbaseProcessor.java
+++ b/nifi-nar-bundles/nifi-couchbase-bundle/nifi-couchbase-processors/src/main/java/org/apache/nifi/processors/couchbase/AbstractCouchbaseProcessor.java
@@ -137,12 +137,10 @@ public abstract class AbstractCouchbaseProcessor extends AbstractProcessor {
}
private CouchbaseClusterControllerService getClusterService(final ProcessContext context) {
- if (clusterService == null) {
- synchronized (AbstractCouchbaseProcessor.class) {
- if (clusterService == null) {
- clusterService = context.getProperty(COUCHBASE_CLUSTER_SERVICE)
+ synchronized (AbstractCouchbaseProcessor.class) {
+ if (clusterService == null) {
+ clusterService = context.getProperty(COUCHBASE_CLUSTER_SERVICE)
.asControllerService(CouchbaseClusterControllerService.class);
- }
}
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/ab794036/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/util/FlowFileEvent.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/util/FlowFileEvent.java b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/util/FlowFileEvent.java
index fdff203..c552d4d 100644
--- a/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/util/FlowFileEvent.java
+++ b/nifi-nar-bundles/nifi-flume-bundle/nifi-flume-processors/src/main/java/org/apache/nifi/processors/flume/util/FlowFileEvent.java
@@ -59,12 +59,8 @@ public class FlowFileEvent implements Event {
@Override
public Map<String, String> getHeaders() {
- if (!headersLoaded) {
- synchronized (headers) {
- if (headersLoaded) {
- return headers;
- }
-
+ synchronized (headers) {
+ if (!headersLoaded) {
headers.putAll(flowFile.getAttributes());
headers.put(ENTRY_DATE_HEADER, Long.toString(flowFile.getEntryDate()));
headers.put(ID_HEADER, Long.toString(flowFile.getId()));
@@ -76,7 +72,6 @@ public class FlowFileEvent implements Event {
}
headers.put(LINEAGE_START_DATE_HEADER, Long.toString(flowFile.getLineageStartDate()));
headers.put(SIZE_HEADER, Long.toString(flowFile.getSize()));
-
headersLoaded = true;
}
}
@@ -94,11 +89,7 @@ public class FlowFileEvent implements Event {
@Override
public byte[] getBody() {
- if (bodyLoaded) {
- return body;
- }
-
- synchronized (bodyLock ) {
+ synchronized (bodyLock) {
if (!bodyLoaded) {
if (flowFile.getSize() > Integer.MAX_VALUE) {
throw new RuntimeException("Can't get body of Event because the backing FlowFile is too large (" + flowFile.getSize() + " bytes)");
[30/50] [abbrv] nifi git commit: NIFI-1174 Refactoring the HBase
client API and adding a PutHBaseJSON which can write a whole row from a
single json document - Adding Complex Field Strategy to PutHBaseJSON to allow
more control of complex fields - Improv
Posted by mc...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/40dd8a0a/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/test/java/org/apache/nifi/hbase/TestHBase_1_1_2_ClientService.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/test/java/org/apache/nifi/hbase/TestHBase_1_1_2_ClientService.java b/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/test/java/org/apache/nifi/hbase/TestHBase_1_1_2_ClientService.java
index 1575f3c..513ea9c 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/test/java/org/apache/nifi/hbase/TestHBase_1_1_2_ClientService.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/test/java/org/apache/nifi/hbase/TestHBase_1_1_2_ClientService.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.nifi.controller.ConfigurationContext;
+import org.apache.nifi.hbase.put.PutColumn;
import org.apache.nifi.hbase.put.PutFlowFile;
import org.apache.nifi.hbase.scan.Column;
import org.apache.nifi.hbase.scan.ResultCell;
@@ -41,6 +42,7 @@ import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
@@ -130,8 +132,9 @@ public class TestHBase_1_1_2_ClientService {
final String columnQualifier = "qualifier1";
final String content = "content1";
- final PutFlowFile putFlowFile = new PutFlowFile(tableName, row, columnFamily, columnQualifier,
- content.getBytes(StandardCharsets.UTF_8), null);
+ final Collection<PutColumn> columns = Collections.singletonList(new PutColumn(columnFamily, columnQualifier,
+ content.getBytes(StandardCharsets.UTF_8)));
+ final PutFlowFile putFlowFile = new PutFlowFile(tableName, row, columns, null);
final TestRunner runner = TestRunners.newTestRunner(TestProcessor.class);
@@ -168,11 +171,13 @@ public class TestHBase_1_1_2_ClientService {
final String content1 = "content1";
final String content2 = "content2";
- final PutFlowFile putFlowFile1 = new PutFlowFile(tableName, row, columnFamily, columnQualifier,
- content1.getBytes(StandardCharsets.UTF_8), null);
+ final Collection<PutColumn> columns1 = Collections.singletonList(new PutColumn(columnFamily, columnQualifier,
+ content1.getBytes(StandardCharsets.UTF_8)));
+ final PutFlowFile putFlowFile1 = new PutFlowFile(tableName, row, columns1, null);
- final PutFlowFile putFlowFile2 = new PutFlowFile(tableName, row, columnFamily, columnQualifier,
- content2.getBytes(StandardCharsets.UTF_8), null);
+ final Collection<PutColumn> columns2 = Collections.singletonList(new PutColumn(columnFamily, columnQualifier,
+ content2.getBytes(StandardCharsets.UTF_8)));
+ final PutFlowFile putFlowFile2 = new PutFlowFile(tableName, row, columns2, null);
final TestRunner runner = TestRunners.newTestRunner(TestProcessor.class);
@@ -214,11 +219,13 @@ public class TestHBase_1_1_2_ClientService {
final String content1 = "content1";
final String content2 = "content2";
- final PutFlowFile putFlowFile1 = new PutFlowFile(tableName, row1, columnFamily, columnQualifier,
- content1.getBytes(StandardCharsets.UTF_8), null);
+ final Collection<PutColumn> columns1 = Collections.singletonList(new PutColumn(columnFamily, columnQualifier,
+ content1.getBytes(StandardCharsets.UTF_8)));
+ final PutFlowFile putFlowFile1 = new PutFlowFile(tableName, row1, columns1, null);
- final PutFlowFile putFlowFile2 = new PutFlowFile(tableName, row2, columnFamily, columnQualifier,
- content2.getBytes(StandardCharsets.UTF_8), null);
+ final Collection<PutColumn> columns2 = Collections.singletonList(new PutColumn(columnFamily, columnQualifier,
+ content2.getBytes(StandardCharsets.UTF_8)));
+ final PutFlowFile putFlowFile2 = new PutFlowFile(tableName, row2, columns2, null);
final TestRunner runner = TestRunners.newTestRunner(TestProcessor.class);
[43/50] [abbrv] nifi git commit: NIFI-655: - Refactoring web security
to use Spring Security Java Configuration. - Introducing security in Web UI
in order to get JWT.
Posted by mc...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/x509/ocsp/OcspCertificateValidator.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/x509/ocsp/OcspCertificateValidator.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/x509/ocsp/OcspCertificateValidator.java
index 81e4bd6..832a63c 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/x509/ocsp/OcspCertificateValidator.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/x509/ocsp/OcspCertificateValidator.java
@@ -42,7 +42,6 @@ import javax.net.ssl.TrustManager;
import javax.net.ssl.TrustManagerFactory;
import javax.net.ssl.X509TrustManager;
import javax.security.auth.x500.X500Principal;
-import javax.servlet.http.HttpServletRequest;
import org.apache.nifi.framework.security.util.SslContextFactory;
import org.apache.nifi.web.security.x509.ocsp.OcspStatus.ValidationStatus;
import org.apache.nifi.web.security.x509.ocsp.OcspStatus.VerificationStatus;
@@ -158,8 +157,7 @@ public class OcspCertificateValidator {
}
/**
- * Loads the trusted certificate authorities according to the specified
- * properties.
+ * Loads the trusted certificate authorities according to the specified properties.
*
* @param properties properties
* @return map of certificate authorities
@@ -208,12 +206,10 @@ public class OcspCertificateValidator {
/**
* Validates the specified certificate using OCSP if configured.
*
- * @param request http request
+ * @param certificates the client certificates
* @throws CertificateStatusException ex
*/
- public void validate(final HttpServletRequest request) throws CertificateStatusException {
- final X509Certificate[] certificates = (X509Certificate[]) request.getAttribute("javax.servlet.request.X509Certificate");
-
+ public void validate(final X509Certificate[] certificates) throws CertificateStatusException {
// only validate if configured to do so
if (client != null && certificates != null && certificates.length > 0) {
final X509Certificate subjectCertificate = getSubjectCertificate(certificates);
@@ -395,13 +391,9 @@ public class OcspCertificateValidator {
}
/**
- * Gets the trusted responder certificate. The response contains the
- * responder certificate, however we cannot blindly trust it. Instead, we
- * use a configured trusted CA. If the responder certificate is a trusted
- * CA, then we can use it. If the responder certificate is not directly
- * trusted, we still may be able to trust it if it was issued by the same CA
- * that issued the subject certificate. Other various checks may be required
- * (this portion is currently not implemented).
+ * Gets the trusted responder certificate. The response contains the responder certificate, however we cannot blindly trust it. Instead, we use a configured trusted CA. If the responder
+ * certificate is a trusted CA, then we can use it. If the responder certificate is not directly trusted, we still may be able to trust it if it was issued by the same CA that issued the subject
+ * certificate. Other various checks may be required (this portion is currently not implemented).
*
* @param responderCertificate cert
* @param issuerCertificate cert
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/resources/nifi-web-security-context.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/resources/nifi-web-security-context.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/resources/nifi-web-security-context.xml
index fa5b9e2..40f678c 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/resources/nifi-web-security-context.xml
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/resources/nifi-web-security-context.xml
@@ -13,70 +13,45 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<!-- marked as lazy so that security beans are not created when applications runs in non-secure mode -->
-<beans default-lazy-init="true"
- xmlns="http://www.springframework.org/schema/beans"
+<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xmlns:sec="http://www.springframework.org/schema/security"
- xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.1.xsd
- http://www.springframework.org/schema/security http://www.springframework.org/schema/security/spring-security-3.2.xsd">
+ xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.1.xsd">
- <!-- http security configuration -->
- <sec:http create-session="stateless" auto-config="false" entry-point-ref="entryPoint">
- <sec:anonymous enabled="false"/>
- <sec:custom-filter ref="nodeAuthorizedUserFilter" before="PRE_AUTH_FILTER"/>
- <sec:custom-filter ref="x509Filter" position="PRE_AUTH_FILTER"/>
- <sec:custom-filter ref="anonymousFilter" position="ANONYMOUS_FILTER"/>
- </sec:http>
+ <!-- certificate extractor -->
+ <bean id="certificateExtractor" class="org.apache.nifi.web.security.x509.X509CertificateExtractor"/>
- <!-- enable method level security -->
- <sec:global-method-security pre-post-annotations="enabled"/>
-
- <!-- entry point reference -->
- <bean id="entryPoint" class="org.apache.nifi.web.security.authentication.NiFiAuthenticationEntryPoint"/>
+ <!-- principal extractor -->
+ <bean id="principalExtractor" class="org.apache.nifi.web.security.x509.SubjectDnX509PrincipalExtractor"/>
- <!-- authentication manager -->
- <sec:authentication-manager alias="authenticationManager">
- <sec:authentication-provider ref="preauthAuthProvider"/>
- </sec:authentication-manager>
-
- <!-- pre-authentication provider -->
- <bean id="preauthAuthProvider" class="org.springframework.security.web.authentication.preauth.PreAuthenticatedAuthenticationProvider">
- <property name="preAuthenticatedUserDetailsService">
- <bean class="org.springframework.security.core.userdetails.UserDetailsByNameServiceWrapper">
- <property name="userDetailsService" ref="userAuthorizationService"/>
- </bean>
- </property>
+ <!-- ocsp validator -->
+ <bean id="ocspValidator" class="org.apache.nifi.web.security.x509.ocsp.OcspCertificateValidator">
+ <constructor-arg ref="nifiProperties"/>
</bean>
-
- <!-- user details service -->
- <bean id="userAuthorizationService" class="org.apache.nifi.web.security.authorization.NiFiAuthorizationService">
- <property name="userService" ref="userService"/>
- <property name="properties" ref="nifiProperties"/>
+
+ <!-- x509 validator -->
+ <bean id="certificateValidator" class="org.apache.nifi.web.security.x509.X509CertificateValidator">
+ <property name="ocspValidator" ref="ocspValidator"/>
</bean>
-
- <!-- performs ocsp certificate validation -->
- <bean id="ocspCertificateValidator" class="org.apache.nifi.web.security.x509.ocsp.OcspCertificateValidator">
- <constructor-arg ref="nifiProperties"/>
+
+ <!-- x509 identity provider -->
+ <bean id="certificateIdentityProvider" class="org.apache.nifi.web.security.x509.X509IdentityProvider">
+ <property name="principalExtractor" ref="principalExtractor"/>
+ <property name="certificateValidator" ref="certificateValidator"/>
</bean>
-
- <!-- custom x509 filter for checking for proxied users -->
- <bean id="x509Filter" class="org.apache.nifi.web.security.x509.X509AuthenticationFilter">
- <property name="authenticationManager" ref="authenticationManager"/>
- <property name="properties" ref="nifiProperties"/>
+
+ <!-- user details service -->
+ <bean id="userDetailsService" class="org.apache.nifi.web.security.authorization.NiFiAuthorizationService">
<property name="userService" ref="userService"/>
- <property name="certificateValidator" ref="ocspCertificateValidator"/>
- <property name="continueFilterChainOnUnsuccessfulAuthentication" value="false"/>
- </bean>
-
- <!-- custom filter for checking for proxied users that are already authenticated -->
- <bean id="nodeAuthorizedUserFilter" class="org.apache.nifi.web.security.authorization.NodeAuthorizedUserFilter">
<property name="properties" ref="nifiProperties"/>
</bean>
-
- <!-- custom anonymous filter to assign default roles based on current operating mode -->
- <bean id="anonymousFilter" class="org.apache.nifi.web.security.anonymous.NiFiAnonymousUserFilter">
- <property name="userService" ref="userService"/>
+
+ <!-- jwt service -->
+ <bean id="jwtService" class="org.apache.nifi.web.security.jwt.JwtService">
+ <constructor-arg ref="keyService"/>
+ </bean>
+
+ <!-- login identity provider -->
+ <bean id="loginIdentityProvider" class="org.apache.nifi.web.security.spring.LoginIdentityProviderFactoryBean">
<property name="properties" ref="nifiProperties"/>
</bean>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/xsd/login-identity-providers.xsd
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/xsd/login-identity-providers.xsd b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/xsd/login-identity-providers.xsd
new file mode 100644
index 0000000..628f390
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/xsd/login-identity-providers.xsd
@@ -0,0 +1,49 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">
+ <!-- role -->
+ <xs:complexType name="Provider">
+ <xs:sequence>
+ <xs:element name="identifier" type="NonEmptyStringType"/>
+ <xs:element name="class" type="NonEmptyStringType"/>
+ <xs:element name="property" type="Property" minOccurs="0" maxOccurs="unbounded" />
+ </xs:sequence>
+ </xs:complexType>
+
+ <!-- Name/Value properties-->
+ <xs:complexType name="Property">
+ <xs:simpleContent>
+ <xs:extension base="xs:string">
+ <xs:attribute name="name" type="NonEmptyStringType"></xs:attribute>
+ </xs:extension>
+ </xs:simpleContent>
+ </xs:complexType>
+
+ <xs:simpleType name="NonEmptyStringType">
+ <xs:restriction base="xs:string">
+ <xs:minLength value="1"/>
+ </xs:restriction>
+ </xs:simpleType>
+
+ <!-- login identity provider -->
+ <xs:element name="loginIdentityProviders">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="provider" type="Provider" minOccurs="0" maxOccurs="unbounded"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+</xs:schema>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/test/java/org/apache/nifi/web/security/authorization/NiFiAuthorizationServiceTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/test/java/org/apache/nifi/web/security/authorization/NiFiAuthorizationServiceTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/test/java/org/apache/nifi/web/security/authorization/NiFiAuthorizationServiceTest.java
index 6d0c3cb..5456552 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/test/java/org/apache/nifi/web/security/authorization/NiFiAuthorizationServiceTest.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/test/java/org/apache/nifi/web/security/authorization/NiFiAuthorizationServiceTest.java
@@ -16,6 +16,7 @@
*/
package org.apache.nifi.web.security.authorization;
+import java.util.Arrays;
import org.apache.nifi.admin.service.AccountDisabledException;
import org.apache.nifi.admin.service.AccountNotFoundException;
import org.apache.nifi.admin.service.AccountPendingException;
@@ -24,8 +25,8 @@ import org.apache.nifi.admin.service.UserService;
import org.apache.nifi.authorization.Authority;
import org.apache.nifi.user.NiFiUser;
import org.apache.nifi.util.NiFiProperties;
-import org.apache.nifi.web.security.DnUtils;
import org.apache.nifi.web.security.UntrustedProxyException;
+import org.apache.nifi.web.security.token.NiFiAuthenticationRequestToken;
import org.apache.nifi.web.security.user.NiFiUserDetails;
import org.junit.Assert;
import org.junit.Before;
@@ -66,10 +67,10 @@ public class NiFiAuthorizationServiceTest {
@Override
public Object answer(InvocationOnMock invocation) throws Throwable {
Object[] args = invocation.getArguments();
- String dn = (String) args[0];
+ String identity = (String) args[0];
- if (null != dn) {
- switch (dn) {
+ if (null != identity) {
+ switch (identity) {
case USER_NOT_FOUND:
case PROXY_NOT_FOUND:
throw new AccountNotFoundException("");
@@ -81,13 +82,13 @@ public class NiFiAuthorizationServiceTest {
throw new AdministrationException();
case USER:
final NiFiUser monitor = new NiFiUser();
- monitor.setDn(dn);
+ monitor.setIdentity(identity);
monitor.getAuthorities().add(Authority.ROLE_MONITOR);
return monitor;
case PROXY:
case PROXY_PROXY:
final NiFiUser proxy = new NiFiUser();
- proxy.setDn(dn);
+ proxy.setIdentity(identity);
proxy.getAuthorities().add(Authority.ROLE_PROXY);
return proxy;
}
@@ -103,15 +104,18 @@ public class NiFiAuthorizationServiceTest {
authorizationService.setUserService(userService);
}
+ private NiFiAuthenticationRequestToken createRequestAuthentication(final String... identities) {
+ return new NiFiAuthenticationRequestToken(Arrays.asList(identities));
+ }
+
/**
- * Ensures the authorization service correctly handles users invalid dn
- * chain.
+ * Ensures the authorization service correctly handles users invalid identity chain.
*
* @throws Exception ex
*/
@Test(expected = UntrustedProxyException.class)
public void testInvalidDnChain() throws Exception {
- authorizationService.loadUserByUsername(USER);
+ authorizationService.loadUserDetails(createRequestAuthentication());
}
/**
@@ -121,7 +125,7 @@ public class NiFiAuthorizationServiceTest {
*/
@Test(expected = UsernameNotFoundException.class)
public void testAccountNotFound() throws Exception {
- authorizationService.loadUserByUsername(DnUtils.formatProxyDn(USER_NOT_FOUND));
+ authorizationService.loadUserDetails(createRequestAuthentication(USER_NOT_FOUND));
}
/**
@@ -131,7 +135,7 @@ public class NiFiAuthorizationServiceTest {
*/
@Test(expected = AccountStatusException.class)
public void testAccountDisabled() throws Exception {
- authorizationService.loadUserByUsername(DnUtils.formatProxyDn(USER_DISABLED));
+ authorizationService.loadUserDetails(createRequestAuthentication(USER_DISABLED));
}
/**
@@ -141,18 +145,17 @@ public class NiFiAuthorizationServiceTest {
*/
@Test(expected = AccountStatusException.class)
public void testAccountPending() throws Exception {
- authorizationService.loadUserByUsername(DnUtils.formatProxyDn(USER_PENDING));
+ authorizationService.loadUserDetails(createRequestAuthentication(USER_PENDING));
}
/**
- * Ensures the authorization service correctly handles account
- * administration exception.
+ * Ensures the authorization service correctly handles account administration exception.
*
* @throws Exception ex
*/
@Test(expected = AuthenticationServiceException.class)
public void testAccountAdminException() throws Exception {
- authorizationService.loadUserByUsername(DnUtils.formatProxyDn(USER_ADMIN_EXCEPTION));
+ authorizationService.loadUserDetails(createRequestAuthentication(USER_ADMIN_EXCEPTION));
}
/**
@@ -162,10 +165,10 @@ public class NiFiAuthorizationServiceTest {
*/
@Test
public void testNoProxy() throws Exception {
- final NiFiUserDetails details = (NiFiUserDetails) authorizationService.loadUserByUsername(DnUtils.formatProxyDn(USER));
+ final NiFiUserDetails details = (NiFiUserDetails) authorizationService.loadUserDetails(createRequestAuthentication(USER));
final NiFiUser user = details.getNiFiUser();
- Assert.assertEquals(USER, user.getDn());
+ Assert.assertEquals(USER, user.getIdentity());
Assert.assertNull(user.getChain());
}
@@ -176,21 +179,18 @@ public class NiFiAuthorizationServiceTest {
*/
@Test(expected = UntrustedProxyException.class)
public void testInvalidProxy() throws Exception {
- final String dnChain = DnUtils.formatProxyDn(USER) + DnUtils.formatProxyDn(USER);
- authorizationService.loadUserByUsername(dnChain);
+ authorizationService.loadUserDetails(createRequestAuthentication(USER, USER));
}
/**
- * Ensures the authorization service correctly handles proxy not found by
- * attempting to create an account request for the proxy.
+ * Ensures the authorization service correctly handles proxy not found by attempting to create an account request for the proxy.
*
* @throws Exception ex
*/
- @Test(expected = UsernameNotFoundException.class)
+ @Test(expected = UntrustedProxyException.class)
public void testProxyNotFound() throws Exception {
try {
- final String dnChain = DnUtils.formatProxyDn(USER) + DnUtils.formatProxyDn(PROXY_NOT_FOUND);
- authorizationService.loadUserByUsername(DnUtils.formatProxyDn(dnChain));
+ authorizationService.loadUserDetails(createRequestAuthentication(USER, PROXY_NOT_FOUND));
} finally {
Mockito.verify(userService).createPendingUserAccount(Mockito.eq(PROXY_NOT_FOUND), Mockito.anyString());
}
@@ -203,19 +203,18 @@ public class NiFiAuthorizationServiceTest {
*/
@Test
public void testProxy() throws Exception {
- final String dnChain = DnUtils.formatProxyDn(USER) + DnUtils.formatProxyDn(PROXY);
- final NiFiUserDetails details = (NiFiUserDetails) authorizationService.loadUserByUsername(dnChain);
+ final NiFiUserDetails details = (NiFiUserDetails) authorizationService.loadUserDetails(createRequestAuthentication(USER, PROXY));
final NiFiUser user = details.getNiFiUser();
// verify the user
- Assert.assertEquals(USER, user.getDn());
+ Assert.assertEquals(USER, user.getIdentity());
Assert.assertNotNull(user.getChain());
// get the proxy
final NiFiUser proxy = user.getChain();
// verify the proxy
- Assert.assertEquals(PROXY, proxy.getDn());
+ Assert.assertEquals(PROXY, proxy.getIdentity());
Assert.assertNull(proxy.getChain());
}
@@ -226,26 +225,25 @@ public class NiFiAuthorizationServiceTest {
*/
@Test
public void testProxyProxy() throws Exception {
- final String dnChain = DnUtils.formatProxyDn(USER) + DnUtils.formatProxyDn(PROXY) + DnUtils.formatProxyDn(PROXY_PROXY);
- final NiFiUserDetails details = (NiFiUserDetails) authorizationService.loadUserByUsername(dnChain);
+ final NiFiUserDetails details = (NiFiUserDetails) authorizationService.loadUserDetails(createRequestAuthentication(USER, PROXY, PROXY_PROXY));
final NiFiUser user = details.getNiFiUser();
// verify the user
- Assert.assertEquals(USER, user.getDn());
+ Assert.assertEquals(USER, user.getIdentity());
Assert.assertNotNull(user.getChain());
// get the proxy
NiFiUser proxy = user.getChain();
// verify the proxy
- Assert.assertEquals(PROXY, proxy.getDn());
+ Assert.assertEquals(PROXY, proxy.getIdentity());
Assert.assertNotNull(proxy.getChain());
// get the proxies proxy
proxy = proxy.getChain();
// verify the proxies proxy
- Assert.assertEquals(PROXY_PROXY, proxy.getDn());
+ Assert.assertEquals(PROXY_PROXY, proxy.getIdentity());
Assert.assertNull(proxy.getChain());
}
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/test/java/org/apache/nifi/web/security/jwt/JwtServiceTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/test/java/org/apache/nifi/web/security/jwt/JwtServiceTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/test/java/org/apache/nifi/web/security/jwt/JwtServiceTest.java
new file mode 100644
index 0000000..59c66ef
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/test/java/org/apache/nifi/web/security/jwt/JwtServiceTest.java
@@ -0,0 +1,445 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.security.jwt;
+
+import io.jsonwebtoken.JwtException;
+import org.apache.commons.codec.CharEncoding;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.nifi.admin.service.AdministrationException;
+import org.apache.nifi.admin.service.KeyService;
+import org.apache.nifi.key.Key;
+import org.apache.nifi.web.security.token.LoginAuthenticationToken;
+import org.codehaus.jettison.json.JSONObject;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.crypto.Mac;
+import javax.crypto.spec.SecretKeySpec;
+import java.io.UnsupportedEncodingException;
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.anyInt;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.when;
+
+public class JwtServiceTest {
+
+ private static final Logger logger = LoggerFactory.getLogger(JwtServiceTest.class);
+
+ /**
+ * These constant strings were generated using the tool at http://jwt.io
+ */
+ private static final String VALID_SIGNED_TOKEN = "eyJhbGciOiJIUzI1NiJ9"
+ + ".eyJzdWIiOiJhbG9wcmVzdG8iLCJpc3MiOiJNb2NrSWRlbnRpdHlQcm92aWRl"
+ + "ciIsImF1ZCI6Ik1vY2tJZGVudGl0eVByb3ZpZGVyIiwicHJlZmVycmVkX3VzZ"
+ + "XJuYW1lIjoiYWxvcHJlc3RvIiwia2lkIjoxLCJleHAiOjI0NDc4MDg3NjEsIm"
+ + "lhdCI6MTQ0NzgwODcwMX0.r6aGZ6FNNYMOpcXW8BK2VYaQeX1uO0Aw1KJfjB3Q1DU";
+
+ // This token has an empty subject field
+ private static final String INVALID_SIGNED_TOKEN = "eyJhbGciOiJIUzI1NiJ9"
+ + ".eyJzdWIiOiIiLCJpc3MiOiJNb2NrSWRlbnRpdHlQcm92aWRlciIsImF1ZCI6Ik1vY2tJZG"
+ + "VudGl0eVByb3ZpZGVyIiwicHJlZmVycmVkX3VzZXJuYW1lIjoiYWxvcHJlc3RvI"
+ + "iwia2lkIjoxLCJleHAiOjI0NDc4MDg3NjEsImlhdCI6MTQ0NzgwODcwMX0"
+ + ".x_1p2M6E0vwWHWMujIUnSL3GkFoDqqICllRxo2SMNaw";
+
+ private static final String VALID_UNSIGNED_TOKEN = "eyJhbGciOiJIUzI1NiJ9"
+ + ".eyJzdWIiOiJhbG9wcmVzdG8iLCJpc3MiOiJNb2NrSWRlbnRpdHlQcm92aWRlciIsImF1ZC"
+ + "I6Ik1vY2tJZGVudGl0eVByb3ZpZGVyIiwicHJlZmVycmVkX3VzZXJuYW1lIjoiYWxvcHJl"
+ + "c3RvIiwia2lkIjoiYWxvcHJlc3RvIiwiZXhwIjoyNDQ3ODA4NzYxLCJpYXQiOjE0NDc4MDg3MDF9";
+
+ // This token has an empty subject field
+ private static final String INVALID_UNSIGNED_TOKEN = "eyJhbGciOiJIUzI1NiJ9"
+ + ".eyJzdWIiOiIiLCJpc3MiOiJNb2NrSWRlbnRpdHlQcm92aWRlciIsImF1ZCI6Ik1vY2tJZGVu"
+ + "dGl0eVByb3ZpZGVyIiwicHJlZmVycmVkX3VzZXJuYW1lIjoiYWxvcHJlc3RvIiwia2lkIjoi"
+ + "YWxvcHJlc3RvIiwiZXhwIjoyNDQ3ODA4NzYxLCJpYXQiOjE0NDc4MDg3MDF9";
+
+ // Algorithm field is "none"
+ private static final String VALID_MALSIGNED_TOKEN = "eyJhbGciOiJub25lIn0"
+ + ".eyJzdWIiOiJhbG9wcmVzdG8iLCJpc3MiOiJNb2NrSWRlbnRpdHlQcm92aWRlciIsImF1ZC"
+ + "I6Ik1vY2tJZGVudGl0eVByb3ZpZGVyIiwicHJlZmVycmVkX3VzZXJuYW1lIjoiYWxvcHJl"
+ + "c3RvIiwia2lkIjoiYWxvcHJlc3RvIiwiZXhwIjoxNDQ3ODA4NzYxLCJpYXQiOjE0NDc4MDg3MDF9"
+ + ".mPO_wMNMl_zjMNevhNvUoXbSJ9Kx6jAe5OxDIAzKQbI";
+
+ // Algorithm field is "none" and no signature is present
+ private static final String VALID_MALSIGNED_NO_SIG_TOKEN = "eyJhbGciOiJub25lIn0"
+ + ".eyJzdWIiOiJhbG9wcmVzdG8iLCJpc3MiOiJNb2NrSWRlbnRpdHlQcm92aWRlciIsImF1ZCI6Ik1vY"
+ + "2tJZGVudGl0eVByb3ZpZGVyIiwicHJlZmVycmVkX3VzZXJuYW1lIjoiYWxvcHJlc3RvIiwia2lkIj"
+ + "oiYWxvcHJlc3RvIiwiZXhwIjoyNDQ3ODA4NzYxLCJpYXQiOjE0NDc4MDg3MDF9.";
+
+ // This token has an empty subject field
+ private static final String INVALID_MALSIGNED_TOKEN = "eyJhbGciOiJIUzI1NiJ9"
+ + ".eyJzdWIiOiIiLCJpc3MiOiJNb2NrSWRlbnRpdHlQcm92aWRlciIsImF1ZCI6Ik1vY2tJZGVud"
+ + "Gl0eVByb3ZpZGVyIiwicHJlZmVycmVkX3VzZXJuYW1lIjoiYWxvcHJlc3RvIiwia2lkIjoiYW"
+ + "xvcHJlc3RvIiwiZXhwIjoxNDQ3ODA4NzYxLCJpYXQiOjE0NDc4MDg3MDF9.WAwmUY4KHKV2oARNodkqDkbZsfRXGZfD2Ccy64GX9QF";
+
+ // This token is signed but expired
+ private static final String EXPIRED_SIGNED_TOKEN = "eyJhbGciOiJIUzI1NiJ9"
+ + ".eyJzdWIiOiIiLCJpc3MiOiJNb2NrSWRlbnRpdHlQcm92aWRlciIsImF1ZCI6Ik"
+ + "1vY2tJZGVudGl0eVByb3ZpZGVyIiwicHJlZmVycmVkX3VzZXJuYW1lIjoiYWxvc"
+ + "HJlc3RvIiwia2lkIjoxLCJleHAiOjE0NDc4MDg3NjEsImlhdCI6MTQ0NzgwODcw"
+ + "MX0.ZPDIhNKuL89vTGXcuztOYaGifwcrQy_gid4j8Sspmto";
+
+ // Subject is "mgilman" but signed with "alopresto" key
+ private static final String IMPOSTER_SIGNED_TOKEN = "eyJhbGciOiJIUzI1NiJ9"
+ + ".eyJzdWIiOiJtZ2lsbWFuIiwiaXNzIjoiTW9ja0lkZW50aXR5UHJvdmlkZXIiLCJ"
+ + "hdWQiOiJNb2NrSWRlbnRpdHlQcm92aWRlciIsInByZWZlcnJlZF91c2VybmFtZSI"
+ + "6ImFsb3ByZXN0byIsImtpZCI6MSwiZXhwIjoyNDQ3ODA4NzYxLCJpYXQiOjE0NDc"
+ + "4MDg3MDF9.aw5OAvLTnb_sHmSQOQzW-A7NImiZgXJ2ngbbNL2Ymkc";
+
+ // Issuer field is set to unknown provider
+ private static final String UNKNOWN_ISSUER_TOKEN = "eyJhbGciOiJIUzI1NiJ9"
+ + ".eyJzdWIiOiJhbG9wcmVzdG8iLCJpc3MiOiJVbmtub3duSWRlbnRpdHlQcm92aWRlciIsIm"
+ + "F1ZCI6Ik1vY2tJZGVudGl0eVByb3ZpZGVyIiwicHJlZmVycmVkX3VzZXJuYW1lIjoiYWxv"
+ + "cHJlc3RvIiwia2lkIjoiYWxvcHJlc3RvIiwiZXhwIjoyNDQ3ODA4NzYxLCJpYXQiOjE0NDc4MDg3MDF9"
+ + ".SAd9tyNwSaijWet9wvAWSNmpxmPSK4XQuLx7h3ARqBo";
+
+ // Issuer field is absent
+ private static final String NO_ISSUER_TOKEN = "eyJhbGciOiJIUzI1NiJ9"
+ + ".eyJzdWIiOiJhbG9wcmVzdG8iLCJhdWQiOiJNb2NrSWRlbnRpdHlQcm92a"
+ + "WRlciIsInByZWZlcnJlZF91c2VybmFtZSI6ImFsb3ByZXN0byIsImtpZCI"
+ + "6MSwiZXhwIjoyNDQ3ODA4NzYxLCJpYXQiOjE0NDc4MDg3MDF9.6kDjDanA"
+ + "g0NQDb3C8FmgbBAYDoIfMAEkF4WMVALsbJA";
+
+ private static final String DEFAULT_HEADER = "{\"alg\":\"HS256\"}";
+ private static final String DEFAULT_IDENTITY = "alopresto";
+
+ private static final String TOKEN_DELIMITER = ".";
+
+ private static final String HMAC_SECRET = "test_hmac_shared_secret";
+
+ private KeyService mockKeyService;
+
+ // Class under test
+ private JwtService jwtService;
+
+ private String generateHS256Token(String rawHeader, String rawPayload, boolean isValid, boolean isSigned) {
+ return generateHS256Token(rawHeader, rawPayload, HMAC_SECRET, isValid, isSigned);
+ }
+
+ private String generateHS256Token(String rawHeader, String rawPayload, String hmacSecret, boolean isValid,
+ boolean isSigned) {
+ try {
+ logger.info("Generating token for " + rawHeader + " + " + rawPayload);
+
+ String base64Header = Base64.encodeBase64URLSafeString(rawHeader.getBytes(CharEncoding.UTF_8));
+ String base64Payload = Base64.encodeBase64URLSafeString(rawPayload.getBytes(CharEncoding.UTF_8));
+ // TODO: Support valid/invalid manipulation
+
+ final String body = base64Header + TOKEN_DELIMITER + base64Payload;
+
+ String signature = generateHMAC(hmacSecret, body);
+
+ return body + TOKEN_DELIMITER + signature;
+ } catch (NoSuchAlgorithmException | InvalidKeyException | UnsupportedEncodingException e) {
+ final String errorMessage = "Could not generate the token";
+ logger.error(errorMessage, e);
+ fail(errorMessage);
+ return null;
+ }
+ }
+
+ private String generateHMAC(String hmacSecret, String body) throws NoSuchAlgorithmException,
+ UnsupportedEncodingException, InvalidKeyException {
+ Mac hmacSHA256 = Mac.getInstance("HmacSHA256");
+ SecretKeySpec secret_key = new SecretKeySpec(hmacSecret.getBytes("UTF-8"), "HmacSHA256");
+ hmacSHA256.init(secret_key);
+ return Base64.encodeBase64URLSafeString(hmacSHA256.doFinal(body.getBytes("UTF-8")));
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ final Key key = new Key();
+ key.setId(1);
+ key.setIdentity(DEFAULT_IDENTITY);
+ key.setKey(HMAC_SECRET);
+
+ mockKeyService = Mockito.mock(KeyService.class);
+ when(mockKeyService.getKey(anyInt())).thenReturn(key);
+ when(mockKeyService.getOrCreateKey(anyString())).thenReturn(key);
+ jwtService = new JwtService(mockKeyService);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+
+ }
+
+ @Test
+ public void testShouldGetAuthenticationForValidToken() throws Exception {
+ // Arrange
+ String token = VALID_SIGNED_TOKEN;
+
+ // Act
+ String identity = jwtService.getAuthenticationFromToken(token);
+ logger.debug("Extracted identity: " + identity);
+
+ // Assert
+ assertEquals("Identity", DEFAULT_IDENTITY, identity);
+ }
+
+ @Test(expected = JwtException.class)
+ public void testShouldNotGetAuthenticationForInvalidToken() throws Exception {
+ // Arrange
+ String token = INVALID_SIGNED_TOKEN;
+
+ // Act
+ String identity = jwtService.getAuthenticationFromToken(token);
+ logger.debug("Extracted identity: " + identity);
+
+ // Assert
+ // Should fail
+ }
+
+ @Test(expected = JwtException.class)
+ public void testShouldNotGetAuthenticationForEmptyToken() throws Exception {
+ // Arrange
+ String token = "";
+
+ // Act
+ String identity = jwtService.getAuthenticationFromToken(token);
+ logger.debug("Extracted identity: " + identity);
+
+ // Assert
+ // Should fail
+ }
+
+ @Test(expected = JwtException.class)
+ public void testShouldNotGetAuthenticationForUnsignedToken() throws Exception {
+ // Arrange
+ String token = VALID_UNSIGNED_TOKEN;
+
+ // Act
+ String identity = jwtService.getAuthenticationFromToken(token);
+ logger.debug("Extracted identity: " + identity);
+
+ // Assert
+ // Should fail
+ }
+
+ @Test(expected = JwtException.class)
+ public void testShouldNotGetAuthenticationForMalsignedToken() throws Exception {
+ // Arrange
+ String token = VALID_MALSIGNED_TOKEN;
+
+ // Act
+ String identity = jwtService.getAuthenticationFromToken(token);
+ logger.debug("Extracted identity: " + identity);
+
+ // Assert
+ // Should fail
+ }
+
+ @Test(expected = JwtException.class)
+ public void testShouldNotGetAuthenticationForTokenWithWrongAlgorithm() throws Exception {
+ // Arrange
+ String token = VALID_MALSIGNED_TOKEN;
+
+ // Act
+ String identity = jwtService.getAuthenticationFromToken(token);
+ logger.debug("Extracted identity: " + identity);
+
+ // Assert
+ // Should fail
+ }
+
+ @Test(expected = JwtException.class)
+ public void testShouldNotGetAuthenticationForTokenWithWrongAlgorithmAndNoSignature() throws Exception {
+ // Arrange
+ String token = VALID_MALSIGNED_NO_SIG_TOKEN;
+
+ // Act
+ String identity = jwtService.getAuthenticationFromToken(token);
+ logger.debug("Extracted identity: " + identity);
+
+ // Assert
+ // Should fail
+ }
+
+ @Ignore("Not yet implemented")
+ @Test(expected = JwtException.class)
+ public void testShouldNotGetAuthenticationForTokenFromUnknownIdentityProvider() throws Exception {
+ // Arrange
+ String token = UNKNOWN_ISSUER_TOKEN;
+
+ // Act
+ String identity = jwtService.getAuthenticationFromToken(token);
+ logger.debug("Extracted identity: " + identity);
+
+ // Assert
+ // Should fail
+ }
+
+ @Test(expected = JwtException.class)
+ public void testShouldNotGetAuthenticationForTokenFromEmptyIdentityProvider() throws Exception {
+ // Arrange
+ String token = NO_ISSUER_TOKEN;
+
+ // Act
+ String identity = jwtService.getAuthenticationFromToken(token);
+ logger.debug("Extracted identity: " + identity);
+
+ // Assert
+ // Should fail
+ }
+
+ @Test(expected = JwtException.class)
+ public void testShouldNotGetAuthenticationForExpiredToken() throws Exception {
+ // Arrange
+ String token = EXPIRED_SIGNED_TOKEN;
+
+ // Act
+ String identity = jwtService.getAuthenticationFromToken(token);
+ logger.debug("Extracted identity: " + identity);
+
+ // Assert
+ // Should fail
+ }
+
+ @Test(expected = JwtException.class)
+ public void testShouldNotGetAuthenticationForImposterToken() throws Exception {
+ // Arrange
+ String token = IMPOSTER_SIGNED_TOKEN;
+
+ // Act
+ String identity = jwtService.getAuthenticationFromToken(token);
+ logger.debug("Extracted identity: " + identity);
+
+ // Assert
+ // Should fail
+ }
+
+ @Test
+ public void testShouldGenerateSignedToken() throws Exception {
+ // Arrange
+
+ // Token expires in 60 seconds
+ final int EXPIRATION_MILLIS = 60000;
+ LoginAuthenticationToken loginAuthenticationToken = new LoginAuthenticationToken("alopresto",
+ EXPIRATION_MILLIS,
+ "MockIdentityProvider");
+ logger.debug("Generating token for " + loginAuthenticationToken);
+
+ final String EXPECTED_HEADER = DEFAULT_HEADER;
+
+ // Convert the expiration time from ms to s
+ final long TOKEN_EXPIRATION_SEC = (long) (loginAuthenticationToken.getExpiration() / 1000.0);
+
+ // Act
+ String token = jwtService.generateSignedToken(loginAuthenticationToken);
+ logger.debug("Generated JWT: " + token);
+
+ // Run after the SUT generates the token to ensure the same issued at time
+ // Split the token, decode the middle section, and form a new String
+ final String DECODED_PAYLOAD = new String(Base64.decodeBase64(token.split("\\.")[1].getBytes()));
+ final long ISSUED_AT_SEC = Long.valueOf(DECODED_PAYLOAD.substring(DECODED_PAYLOAD.lastIndexOf(":") + 1,
+ DECODED_PAYLOAD.length() - 1));
+ logger.trace("Actual token was issued at " + ISSUED_AT_SEC);
+
+ // Always use LinkedHashMap to enforce order of the keys because the signature depends on order
+ Map<String, Object> claims = new LinkedHashMap<>();
+ claims.put("sub", "alopresto");
+ claims.put("iss", "MockIdentityProvider");
+ claims.put("aud", "MockIdentityProvider");
+ claims.put("preferred_username", "alopresto");
+ claims.put("kid", 1);
+ claims.put("exp", TOKEN_EXPIRATION_SEC);
+ claims.put("iat", ISSUED_AT_SEC);
+ logger.trace("JSON Object to String: " + new JSONObject(claims).toString());
+
+ final String EXPECTED_PAYLOAD = new JSONObject(claims).toString();
+ final String EXPECTED_TOKEN_STRING = generateHS256Token(EXPECTED_HEADER, EXPECTED_PAYLOAD, true, true);
+ logger.debug("Expected JWT: " + EXPECTED_TOKEN_STRING);
+
+ // Assert
+ assertEquals("JWT token", EXPECTED_TOKEN_STRING, token);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testShouldNotGenerateTokenWithNullAuthenticationToken() throws Exception {
+ // Arrange
+ LoginAuthenticationToken nullLoginAuthenticationToken = null;
+ logger.debug("Generating token for " + nullLoginAuthenticationToken);
+
+ // Act
+ jwtService.generateSignedToken(nullLoginAuthenticationToken);
+
+ // Assert
+ // Should throw exception
+ }
+
+ @Test(expected = JwtException.class)
+ public void testShouldNotGenerateTokenWithEmptyIdentity() throws Exception {
+ // Arrange
+ final int EXPIRATION_MILLIS = 60000;
+ LoginAuthenticationToken emptyIdentityLoginAuthenticationToken = new LoginAuthenticationToken("",
+ EXPIRATION_MILLIS, "MockIdentityProvider");
+ logger.debug("Generating token for " + emptyIdentityLoginAuthenticationToken);
+
+ // Act
+ jwtService.generateSignedToken(emptyIdentityLoginAuthenticationToken);
+
+ // Assert
+ // Should throw exception
+ }
+
+ @Test(expected = JwtException.class)
+ public void testShouldNotGenerateTokenWithNullIdentity() throws Exception {
+ // Arrange
+ final int EXPIRATION_MILLIS = 60000;
+ LoginAuthenticationToken nullIdentityLoginAuthenticationToken = new LoginAuthenticationToken(null,
+ EXPIRATION_MILLIS, "MockIdentityProvider");
+ logger.debug("Generating token for " + nullIdentityLoginAuthenticationToken);
+
+ // Act
+ jwtService.generateSignedToken(nullIdentityLoginAuthenticationToken);
+
+ // Assert
+ // Should throw exception
+ }
+
+ @Test(expected = JwtException.class)
+ public void testShouldNotGenerateTokenWithMissingKey() throws Exception {
+ // Arrange
+ final int EXPIRATION_MILLIS = 60000;
+ LoginAuthenticationToken loginAuthenticationToken = new LoginAuthenticationToken("alopresto",
+ EXPIRATION_MILLIS,
+ "MockIdentityProvider");
+ logger.debug("Generating token for " + loginAuthenticationToken);
+
+ // Set up the bad key service
+ KeyService missingKeyService = Mockito.mock(KeyService.class);
+ when(missingKeyService.getOrCreateKey(anyString())).thenThrow(new AdministrationException("Could not find a "
+ + "key for that user"));
+ jwtService = new JwtService(missingKeyService);
+
+ // Act
+ jwtService.generateSignedToken(loginAuthenticationToken);
+
+ // Assert
+ // Should throw exception
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/test/resources/logback-test.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/test/resources/logback-test.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/test/resources/logback-test.xml
new file mode 100644
index 0000000..fe5d27e
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/test/resources/logback-test.xml
@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+ <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+ <pattern>%-4r [%t] %-5p %c - %m%n</pattern>
+ </encoder>
+ </appender>
+
+ <appender name="FILE" class="ch.qos.logback.core.FileAppender">
+ <file>./target/log</file>
+ <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+ <pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
+ </encoder>
+ </appender>
+
+
+ <logger name="org.apache.nifi" level="TRACE"/>
+ <root level="INFO">
+ <appender-ref ref="CONSOLE"/>
+ </root>
+</configuration>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/pom.xml
index 195f14b..a4ef516 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/pom.xml
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/pom.xml
@@ -32,6 +32,7 @@
<templates.filter>templates.properties</templates.filter>
<users.filter>users.properties</users.filter>
<bulletin.board.filter>bulletin-board.properties</bulletin.board.filter>
+ <login.filter>login.properties</login.filter>
<provenance.filter>provenance.properties</provenance.filter>
</properties>
<build>
@@ -54,6 +55,7 @@
<filter>src/main/resources/filters/${templates.filter}</filter>
<filter>src/main/resources/filters/${users.filter}</filter>
<filter>src/main/resources/filters/${bulletin.board.filter}</filter>
+ <filter>src/main/resources/filters/${login.filter}</filter>
<filter>src/main/resources/filters/${provenance.filter}</filter>
</filters>
<plugins>
@@ -89,7 +91,8 @@
**/cluster.jsp,
**/templates.jsp,
**/users.jsp,
- **/bulletin-board.jsp
+ **/bulletin-board.jsp,
+ **/login.jsp
</excludes>
</configuration>
</execution>
@@ -209,6 +212,14 @@
</includes>
<filtering>true</filtering>
</resource>
+ <resource>
+ <directory>src/main/webapp/WEB-INF/pages</directory>
+ <targetPath>WEB-INF/pages</targetPath>
+ <includes>
+ <include>login.jsp</include>
+ </includes>
+ <filtering>true</filtering>
+ </resource>
</webResources>
</configuration>
</plugin>
@@ -229,6 +240,7 @@
<templates.filter>templates-min.properties</templates.filter>
<users.filter>users-min.properties</users.filter>
<bulletin.board.filter>bulletin-board-min.properties</bulletin.board.filter>
+ <login.filter>login-min.properties</login.filter>
<provenance.filter>provenance-min.properties</provenance.filter>
</properties>
<build>
@@ -263,11 +275,10 @@
<include>${staging.dir}/js/nf/canvas/nf-canvas-utils.js</include>
<include>${staging.dir}/js/nf/nf-dialog.js</include>
<include>${staging.dir}/js/nf/nf-shell.js</include>
- <include>${staging.dir}/js/nf/canvas/nf-storage.js</include>
+ <include>${staging.dir}/js/nf/nf-storage.js</include>
<include>${staging.dir}/js/nf/canvas/nf-snippet.js</include>
<include>${staging.dir}/js/nf/canvas/nf-canvas-toolbox.js</include>
<include>${staging.dir}/js/nf/canvas/nf-custom-ui.js</include>
- <include>${staging.dir}/js/nf/canvas/nf-registration.js</include>
<include>${staging.dir}/js/nf/canvas/nf-controller-service.js</include>
<include>${staging.dir}/js/nf/canvas/nf-reporting-task.js</include>
<include>${staging.dir}/js/nf/canvas/nf-processor-configuration.js</include>
@@ -317,6 +328,7 @@
<include>${staging.dir}/js/nf/nf-client.js</include>
<include>${staging.dir}/js/nf/nf-common.js</include>
<include>${staging.dir}/js/nf/nf-dialog.js</include>
+ <include>${staging.dir}/js/nf/nf-storage.js</include>
<include>${staging.dir}/js/nf/history/nf-history.js</include>
<include>${staging.dir}/js/nf/history/nf-history-table.js</include>
<include>${staging.dir}/js/nf/history/nf-history-model.js</include>
@@ -329,6 +341,7 @@
<include>${staging.dir}/js/nf/nf-client.js</include>
<include>${staging.dir}/js/nf/nf-common.js</include>
<include>${staging.dir}/js/nf/nf-dialog.js</include>
+ <include>${staging.dir}/js/nf/nf-storage.js</include>
<include>${staging.dir}/js/nf/provenance/nf-provenance.js</include>
<include>${staging.dir}/js/nf/provenance/nf-provenance-table.js</include>
</includes>
@@ -340,6 +353,7 @@
<include>${staging.dir}/js/nf/nf-client.js</include>
<include>${staging.dir}/js/nf/nf-common.js</include>
<include>${staging.dir}/js/nf/nf-dialog.js</include>
+ <include>${staging.dir}/js/nf/nf-storage.js</include>
<include>${staging.dir}/js/nf/nf-processor-details.js</include>
<include>${staging.dir}/js/nf/nf-connection-details.js</include>
<include>${staging.dir}/js/nf/summary/nf-summary.js</include>
@@ -354,6 +368,7 @@
<include>${staging.dir}/js/nf/nf-client.js</include>
<include>${staging.dir}/js/nf/nf-common.js</include>
<include>${staging.dir}/js/nf/nf-dialog.js</include>
+ <include>${staging.dir}/js/nf/nf-storage.js</include>
<include>${staging.dir}/js/nf/counters/nf-counters.js</include>
<include>${staging.dir}/js/nf/counters/nf-counters-table.js</include>
</includes>
@@ -365,6 +380,7 @@
<include>${staging.dir}/js/nf/nf-client.js</include>
<include>${staging.dir}/js/nf/nf-common.js</include>
<include>${staging.dir}/js/nf/nf-dialog.js</include>
+ <include>${staging.dir}/js/nf/nf-storage.js</include>
<include>${staging.dir}/js/nf/templates/nf-templates.js</include>
<include>${staging.dir}/js/nf/templates/nf-templates-table.js</include>
</includes>
@@ -376,6 +392,7 @@
<include>${staging.dir}/js/nf/nf-client.js</include>
<include>${staging.dir}/js/nf/nf-common.js</include>
<include>${staging.dir}/js/nf/nf-dialog.js</include>
+ <include>${staging.dir}/js/nf/nf-storage.js</include>
<include>${staging.dir}/js/nf/cluster/nf-cluster.js</include>
<include>${staging.dir}/js/nf/cluster/nf-cluster-table.js</include>
</includes>
@@ -387,6 +404,7 @@
<include>${staging.dir}/js/nf/nf-client.js</include>
<include>${staging.dir}/js/nf/nf-common.js</include>
<include>${staging.dir}/js/nf/nf-dialog.js</include>
+ <include>${staging.dir}/js/nf/nf-storage.js</include>
<include>${staging.dir}/js/nf/users/nf-users.js</include>
<include>${staging.dir}/js/nf/users/nf-users-table.js</include>
</includes>
@@ -398,11 +416,23 @@
<include>${staging.dir}/js/nf/nf-client.js</include>
<include>${staging.dir}/js/nf/nf-common.js</include>
<include>${staging.dir}/js/nf/nf-dialog.js</include>
+ <include>${staging.dir}/js/nf/nf-storage.js</include>
<include>${staging.dir}/js/nf/bulletin-board/nf-bulletin-board.js</include>
</includes>
</aggregation>
<aggregation>
<insertNewLine>true</insertNewLine>
+ <output>${project.build.directory}/${project.build.finalName}/js/nf/login/nf-login-all.js</output>
+ <includes>
+ <include>${staging.dir}/js/nf/nf-client.js</include>
+ <include>${staging.dir}/js/nf/nf-common.js</include>
+ <include>${staging.dir}/js/nf/nf-dialog.js</include>
+ <include>${staging.dir}/js/nf/nf-storage.js</include>
+ <include>${staging.dir}/js/nf/login/nf-login.js</include>
+ </includes>
+ </aggregation>
+ <aggregation>
+ <insertNewLine>true</insertNewLine>
<output>${project.build.directory}/${project.build.finalName}/css/nf-canvas-all.css</output>
<includes>
<include>${staging.dir}/css/reporting-task.css</include>
@@ -418,7 +448,6 @@
<include>${staging.dir}/css/connection-configuration.css</include>
<include>${staging.dir}/css/connection-details.css</include>
<include>${staging.dir}/css/shell.css</include>
- <include>${staging.dir}/css/registration.css</include>
<include>${staging.dir}/css/dialog.css</include>
<include>${staging.dir}/css/new-processor-dialog.css</include>
<include>${staging.dir}/css/new-controller-service-dialog.css</include>
@@ -517,6 +546,16 @@
<include>${staging.dir}/css/bulletin-board.css</include>
</includes>
</aggregation>
+ <aggregation>
+ <insertNewLine>true</insertNewLine>
+ <output>${project.build.directory}/${project.build.finalName}/css/nf-login-all.css</output>
+ <includes>
+ <include>${staging.dir}/css/main.css</include>
+ <include>${staging.dir}/css/banner.css</include>
+ <include>${staging.dir}/css/dialog.css</include>
+ <include>${staging.dir}/css/login.css</include>
+ </includes>
+ </aggregation>
</aggregations>
</configuration>
</execution>
@@ -556,6 +595,8 @@
css/nf-users-all.css.gz,
css/nf-bulletin-board-all.css,
css/nf-bulletin-board-all.css.gz,
+ css/nf-login-all.css,
+ css/nf-login-all.css.gz,
js/*,
js/d3/**/*,
js/codemirror/**/*,
@@ -584,6 +625,8 @@
js/nf/users/nf-users-all.js.gz,
js/nf/bulletin-board/nf-bulletin-board-all.js,
js/nf/bulletin-board/nf-bulletin-board-all.js.gz,
+ js/nf/login/nf-login-all.js,
+ js/nf/login/nf-login-all.js.gz,
images/*,
resources/*,
images/*,
@@ -615,7 +658,6 @@
the application classpath or the maven jetty plugin classpath defined
above.
-->
-
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
@@ -651,6 +693,5 @@
<artifactId>javax.servlet.jsp.jstl-api</artifactId>
<scope>provided</scope>
</dependency>
-
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/META-INF/LICENSE
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/META-INF/LICENSE b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/META-INF/LICENSE
index 40d0725..6769105 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/META-INF/LICENSE
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/META-INF/LICENSE
@@ -351,6 +351,28 @@ For details see http://jqueryui.com
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+This product bundles 'jquery.base64.js' which is available under an MIT style license.
+
+ Copyright (c) 2013 Yannick Albert (http://yckart.com/)
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the "Software"),
+ to deal in the Software without restriction, including without limitation
+ the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ sell copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
This product bundles 'SlickGrid v2.2' which is available under an MIT style license.
Copyright (c) 2010 Michael Leibman, http://github.com/mleibman/slickgrid
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/bulletin-board.properties
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/bulletin-board.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/bulletin-board.properties
index 55df244..96a550f 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/bulletin-board.properties
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/bulletin-board.properties
@@ -16,6 +16,7 @@
nf.bulletin.board.script.tags=<script type="text/javascript" src="js/nf/nf-namespace.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/nf-common.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/nf-dialog.js?${project.version}"></script>\n\
+<script type="text/javascript" src="js/nf/nf-storage.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/bulletin-board/nf-bulletin-board.js?${project.version}"></script>
nf.bulletin.board.style.tags=<link rel="stylesheet" href="css/reset.css?${project.version}" type="text/css" />\n\
<link rel="stylesheet" href="css/main.css?${project.version}" type="text/css" />\n\
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/canvas.properties
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/canvas.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/canvas.properties
index fd2bc17..bf61846 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/canvas.properties
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/canvas.properties
@@ -19,11 +19,10 @@ nf.canvas.script.tags=<script type="text/javascript" src="js/nf/nf-namespace.js?
<script type="text/javascript" src="js/nf/canvas/nf-canvas-utils.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/nf-dialog.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/nf-shell.js?${project.version}"></script>\n\
-<script type="text/javascript" src="js/nf/canvas/nf-storage.js?${project.version}"></script>\n\
+<script type="text/javascript" src="js/nf/nf-storage.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/canvas/nf-snippet.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/canvas/nf-canvas-toolbox.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/canvas/nf-custom-ui.js?${project.version}"></script>\n\
-<script type="text/javascript" src="js/nf/canvas/nf-registration.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/canvas/nf-controller-service.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/canvas/nf-reporting-task.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/canvas/nf-processor-configuration.js?${project.version}"></script>\n\
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/cluster.properties
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/cluster.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/cluster.properties
index f1481d8..9327be6 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/cluster.properties
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/cluster.properties
@@ -16,6 +16,7 @@
nf.cluster.script.tags=<script type="text/javascript" src="js/nf/nf-namespace.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/nf-common.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/nf-dialog.js?${project.version}"></script>\n\
+<script type="text/javascript" src="js/nf/nf-storage.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/cluster/nf-cluster.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/cluster/nf-cluster-table.js?${project.version}"></script>
nf.cluster.style.tags=<link rel="stylesheet" href="css/reset.css?${project.version}" type="text/css" />\n\
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/counters.properties
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/counters.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/counters.properties
index 8b07e57..be22f55 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/counters.properties
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/counters.properties
@@ -16,6 +16,7 @@
nf.counters.script.tags=<script type="text/javascript" src="js/nf/nf-namespace.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/nf-common.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/nf-dialog.js?${project.version}"></script>\n\
+<script type="text/javascript" src="js/nf/nf-storage.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/counters/nf-counters.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/counters/nf-counters-table.js?${project.version}"></script>
nf.counters.style.tags=<link rel="stylesheet" href="css/reset.css?${project.version}" type="text/css" />\n\
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/history.properties
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/history.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/history.properties
index 818ee58..9bbe640 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/history.properties
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/history.properties
@@ -16,6 +16,7 @@
nf.history.script.tags=<script type="text/javascript" src="js/nf/nf-namespace.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/nf-common.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/nf-dialog.js?${project.version}"></script>\n\
+<script type="text/javascript" src="js/nf/nf-storage.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/history/nf-history.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/history/nf-history-table.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/history/nf-history-model.js?${project.version}"></script>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/login-min.properties
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/login-min.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/login-min.properties
new file mode 100644
index 0000000..4dafb02
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/login-min.properties
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+nf.login.script.tags=<script type="text/javascript" src="js/nf/login/nf-login-all.js?${project.version}"></script>
+nf.login.style.tags=<link rel="stylesheet" href="css/nf-login-all.css?${project.version}" type="text/css" />\n\
+<link rel="stylesheet" href="css/message-pane.css?${project.version}" type="text/css" />
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/login.properties
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/login.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/login.properties
new file mode 100644
index 0000000..28ff4b9
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/login.properties
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+nf.login.script.tags=<script type="text/javascript" src="js/nf/nf-common.js?${project.version}"></script>\n\
+<script type="text/javascript" src="js/nf/nf-dialog.js?${project.version}"></script>\n\
+<script type="text/javascript" src="js/nf/nf-storage.js?${project.version}"></script>\n\
+<script type="text/javascript" src="js/nf/login/nf-login.js?${project.version}"></script>
+nf.login.style.tags=<link rel="stylesheet" href="css/reset.css?${project.version}" type="text/css" />\n\
+<link rel="stylesheet" href="css/main.css?${project.version}" type="text/css" />\n\
+<link rel="stylesheet" href="css/banner.css?${project.version}" type="text/css" />\n\
+<link rel="stylesheet" href="css/dialog.css?${project.version}" type="text/css" />\n\
+<link rel="stylesheet" href="css/message-pane.css?${project.version}" type="text/css" />\n\
+<link rel="stylesheet" href="css/login.css?${project.version}" type="text/css" />
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/provenance.properties
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/provenance.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/provenance.properties
index 655d3bf..daa7c43 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/provenance.properties
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/provenance.properties
@@ -16,6 +16,7 @@
nf.provenance.script.tags=<script type="text/javascript" src="js/nf/nf-namespace.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/nf-common.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/nf-dialog.js?${project.version}"></script>\n\
+<script type="text/javascript" src="js/nf/nf-storage.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/provenance/nf-provenance.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/provenance/nf-provenance-table.js?${project.version}"></script>
nf.provenance.style.tags=<link rel="stylesheet" href="css/reset.css?${project.version}" type="text/css" />\n\
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/summary.properties
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/summary.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/summary.properties
index 90d35d9..2417ade 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/summary.properties
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/summary.properties
@@ -16,6 +16,7 @@
nf.summary.script.tags=<script type="text/javascript" src="js/nf/nf-namespace.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/nf-common.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/nf-dialog.js?${project.version}"></script>\n\
+<script type="text/javascript" src="js/nf/nf-storage.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/nf-processor-details.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/nf-connection-details.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/summary/nf-summary.js?${project.version}"></script>\n\
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/templates.properties
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/templates.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/templates.properties
index 79c59e7..007bd69 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/templates.properties
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/templates.properties
@@ -16,6 +16,7 @@
nf.templates.script.tags=<script type="text/javascript" src="js/nf/nf-namespace.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/nf-common.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/nf-dialog.js?${project.version}"></script>\n\
+<script type="text/javascript" src="js/nf/nf-storage.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/templates/nf-templates.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/templates/nf-templates-table.js?${project.version}"></script>
nf.templates.style.tags=<link rel="stylesheet" href="css/reset.css?${project.version}" type="text/css" />\n\
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/users.properties
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/users.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/users.properties
index a4e586a..9824a8b 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/users.properties
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/users.properties
@@ -16,6 +16,7 @@
nf.users.script.tags=<script type="text/javascript" src="js/nf/nf-namespace.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/nf-common.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/nf-dialog.js?${project.version}"></script>\n\
+<script type="text/javascript" src="js/nf/nf-storage.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/users/nf-users.js?${project.version}"></script>\n\
<script type="text/javascript" src="js/nf/users/nf-users-table.js?${project.version}"></script>
nf.users.style.tags=<link rel="stylesheet" href="css/reset.css?${project.version}" type="text/css" />\n\
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/bulletin-board.jsp
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/bulletin-board.jsp b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/bulletin-board.jsp
index 6df0e51..3eaa225 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/bulletin-board.jsp
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/bulletin-board.jsp
@@ -28,6 +28,7 @@
<link rel="stylesheet" href="js/jquery/qtip2/jquery.qtip.min.css?" type="text/css" />
<link rel="stylesheet" href="js/jquery/ui-smoothness/jquery-ui-1.10.4.min.css" type="text/css" />
<script type="text/javascript" src="js/jquery/jquery-2.1.1.min.js"></script>
+ <script type="text/javascript" src="js/jquery/jquery.base64.js"></script>
<script type="text/javascript" src="js/jquery/jquery.center.js"></script>
<script type="text/javascript" src="js/jquery/combo/jquery.combo.js?${project.version}"></script>
<script type="text/javascript" src="js/jquery/modal/jquery.modal.js?${project.version}"></script>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/canvas.jsp
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/canvas.jsp b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/canvas.jsp
index 1bf63a6..f9970df 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/canvas.jsp
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/canvas.jsp
@@ -40,8 +40,8 @@
<script type="text/javascript" src="js/codemirror/lib/codemirror-compressed.js"></script>
<script type="text/javascript" src="js/jquery/jquery-2.1.1.min.js"></script>
<script type="text/javascript" src="js/jquery/ui-smoothness/jquery-ui-1.10.4.min.js"></script>
+ <script type="text/javascript" src="js/jquery/jquery.base64.js"></script>
<script type="text/javascript" src="js/jquery/jquery.center.js"></script>
- <script type="text/javascript" src="js/jquery/jquery.count.js"></script>
<script type="text/javascript" src="js/jquery/jquery.ellipsis.js"></script>
<script type="text/javascript" src="js/jquery/jquery.each.js"></script>
<script type="text/javascript" src="js/jquery/jquery.tab.js"></script>
@@ -73,7 +73,6 @@
<img id="splash-img" src="images/loadAnimation.gif" alt="Loading..."/>
</div>
<jsp:include page="/WEB-INF/partials/message-pane.jsp"/>
- <jsp:include page="/WEB-INF/partials/canvas/registration.jsp"/>
<jsp:include page="/WEB-INF/partials/banners-main.jsp"/>
<jsp:include page="/WEB-INF/partials/canvas/canvas-header.jsp"/>
<jsp:include page="/WEB-INF/partials/canvas/about-dialog.jsp"/>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/cluster.jsp
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/cluster.jsp b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/cluster.jsp
index 55ffabe..acf2e55 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/cluster.jsp
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/cluster.jsp
@@ -30,6 +30,7 @@
<link rel="stylesheet" href="js/jquery/slickgrid/css/slick.grid.css" type="text/css" />
<link rel="stylesheet" href="js/jquery/slickgrid/css/slick-default-theme.css" type="text/css" />
<script type="text/javascript" src="js/jquery/jquery-2.1.1.min.js"></script>
+ <script type="text/javascript" src="js/jquery/jquery.base64.js"></script>
<script type="text/javascript" src="js/jquery/jquery.center.js"></script>
<script type="text/javascript" src="js/jquery/combo/jquery.combo.js?${project.version}"></script>
<script type="text/javascript" src="js/jquery/modal/jquery.modal.js?${project.version}"></script>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/counters.jsp
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/counters.jsp b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/counters.jsp
index 7362cf2..e3113f9 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/counters.jsp
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/counters.jsp
@@ -30,6 +30,7 @@
<link rel="stylesheet" href="js/jquery/slickgrid/css/slick.grid.css" type="text/css" />
<link rel="stylesheet" href="js/jquery/slickgrid/css/slick-default-theme.css" type="text/css" />
<script type="text/javascript" src="js/jquery/jquery-2.1.1.min.js"></script>
+ <script type="text/javascript" src="js/jquery/jquery.base64.js"></script>
<script type="text/javascript" src="js/jquery/jquery.center.js"></script>
<script type="text/javascript" src="js/jquery/combo/jquery.combo.js?${project.version}"></script>
<script type="text/javascript" src="js/jquery/modal/jquery.modal.js?${project.version}"></script>
http://git-wip-us.apache.org/repos/asf/nifi/blob/aaf14c45/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/history.jsp
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/history.jsp b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/history.jsp
index 935512a..ac28bb7 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/history.jsp
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/history.jsp
@@ -30,6 +30,7 @@
<link rel="stylesheet" href="js/jquery/slickgrid/css/slick.grid.css" type="text/css" />
<link rel="stylesheet" href="js/jquery/slickgrid/css/slick-default-theme.css" type="text/css" />
<script type="text/javascript" src="js/jquery/jquery-2.1.1.min.js"></script>
+ <script type="text/javascript" src="js/jquery/jquery.base64.js"></script>
<script type="text/javascript" src="js/jquery/jquery.center.js"></script>
<script type="text/javascript" src="js/jquery/modal/jquery.modal.js?${project.version}"></script>
<script type="text/javascript" src="js/jquery/combo/jquery.combo.js?${project.version}"></script>