You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by jo...@apache.org on 2018/11/21 15:06:01 UTC
nifi git commit: NIFI-5833 This closes #3180. Marked GetTwitter
Consumer Key and Access Token processor properties as sensitive. NIFI-5833
Added unit test to demonstrate arbitrary decryption of sensitive values
regardless of processor property sensitive
Repository: nifi
Updated Branches:
refs/heads/master 3c7012ffd -> f6b171d5f
NIFI-5833 This closes #3180. Marked GetTwitter Consumer Key and Access Token processor properties as sensitive.
NIFI-5833 Added unit test to demonstrate arbitrary decryption of sensitive values regardless of processor property sensitive status.
NIFI-5833 Updated GetTwitter documentation with note about 1.9.0+ marking Consumer Key and Access Token as sensitive.
Signed-off-by: joewitt <jo...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/f6b171d5
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/f6b171d5
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/f6b171d5
Branch: refs/heads/master
Commit: f6b171d5f76aabf59c1e4644927ab6484be9839a
Parents: 3c7012f
Author: Andy LoPresto <al...@apache.org>
Authored: Tue Nov 20 18:15:40 2018 -0800
Committer: joewitt <jo...@apache.org>
Committed: Wed Nov 21 10:02:25 2018 -0500
----------------------------------------------------------------------
.../serialization/FlowFromDOMFactoryTest.groovy | 82 ++++++++++++++++++++
.../nifi/processors/twitter/GetTwitter.java | 37 +++++----
2 files changed, 100 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/f6b171d5/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/groovy/org/apache/nifi/controller/serialization/FlowFromDOMFactoryTest.groovy
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/groovy/org/apache/nifi/controller/serialization/FlowFromDOMFactoryTest.groovy b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/groovy/org/apache/nifi/controller/serialization/FlowFromDOMFactoryTest.groovy
index 4beae84..8f82d3a 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/groovy/org/apache/nifi/controller/serialization/FlowFromDOMFactoryTest.groovy
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/groovy/org/apache/nifi/controller/serialization/FlowFromDOMFactoryTest.groovy
@@ -17,12 +17,14 @@
package org.apache.nifi.controller.serialization
import org.apache.commons.codec.binary.Hex
+import org.apache.nifi.controller.StandardFlowSynchronizer
import org.apache.nifi.encrypt.EncryptionException
import org.apache.nifi.encrypt.StringEncryptor
import org.apache.nifi.properties.StandardNiFiProperties
import org.apache.nifi.security.kms.CryptoUtils
import org.apache.nifi.security.util.EncryptionMethod
import org.apache.nifi.util.NiFiProperties
+import org.apache.nifi.web.api.dto.ProcessGroupDTO
import org.bouncycastle.jce.provider.BouncyCastleProvider
import org.junit.After
import org.junit.Before
@@ -32,12 +34,15 @@ import org.junit.runner.RunWith
import org.junit.runners.JUnit4
import org.slf4j.Logger
import org.slf4j.LoggerFactory
+import org.w3c.dom.Document
+import org.w3c.dom.Element
import javax.crypto.Cipher
import javax.crypto.SecretKey
import javax.crypto.SecretKeyFactory
import javax.crypto.spec.PBEKeySpec
import javax.crypto.spec.PBEParameterSpec
+import java.nio.charset.StandardCharsets
import java.security.Security
import static groovy.test.GroovyAssert.shouldFail
@@ -135,6 +140,83 @@ class FlowFromDOMFactoryTest {
assert msg.message =~ "Check that the ${KEY} value in nifi.properties matches the value used to encrypt the flow.xml.gz file"
}
+ @Test
+ void testShouldDecryptSensitiveFlowValueRegardlessOfPropertySensitiveStatus() throws Exception {
+ // Arrange
+
+ // Create a mock Element object to be parsed
+
+ // TODO: Mock call to StandardFlowSynchronizer#readFlowFromDisk()
+ final String FLOW_XML = """<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<flowController encoding-version="1.3">
+ <maxTimerDrivenThreadCount>10</maxTimerDrivenThreadCount>
+ <maxEventDrivenThreadCount>5</maxEventDrivenThreadCount>
+ <registries/>
+ <rootGroup>
+ <id>32aeba59-0167-1000-fc76-847bf5d10d73</id>
+ <name>NiFi Flow</name>
+ <position x="0.0" y="0.0"/>
+ <comment/>
+ <processor>
+ <id>32af5e4e-0167-1000-ad5f-c79ff57c851e</id>
+ <name>Example Processor</name>
+ <position x="461.0" y="80.0"/>
+ <styles/>
+ <comment/>
+ <class>org.apache.nifi.processors.test.ExampleProcessor</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-test-nar</artifact>
+ <version>1.9.0-SNAPSHOT</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>STOPPED</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>Plaintext Property</name>
+ <value>plain value</value>
+ </property>
+ <property>
+ <name>Sensitive Property</name>
+ <value>enc{29077eedc9af7515cc3e0d2d29a93a5cbb059164876948458fd0c890009c8661}</value>
+ </property>
+ </processor>
+ </rootGroup>
+ <controllerServices/>
+ <reportingTasks/>
+</flowController>
+"""
+
+ // TODO: Mock call to StandardFlowSynchronizer#parseFlowBytes()
+ Document flow = StandardFlowSynchronizer.parseFlowBytes(FLOW_XML.getBytes(StandardCharsets.UTF_8))
+
+ // Logic to extract root process group
+ final Element rootElement = flow.getDocumentElement()
+
+ final Element rootGroupElement = (Element) rootElement.getElementsByTagName("rootGroup").item(0)
+ final FlowEncodingVersion encodingVersion = FlowEncodingVersion.parse(rootGroupElement)
+
+ StringEncryptor flowEncryptor = new StringEncryptor(EncryptionMethod.MD5_128AES.algorithm, EncryptionMethod.MD5_128AES.provider, DEFAULT_PASSWORD)
+
+ // Act
+ ProcessGroupDTO decryptedProcessGroupDTO = FlowFromDOMFactory.getProcessGroup(null, rootGroupElement, flowEncryptor, encodingVersion)
+ logger.info("PG DTO: ${decryptedProcessGroupDTO}")
+
+ // Assert
+ def processorProperties = decryptedProcessGroupDTO.contents.processors.first().config.properties
+ logger.info("Parsed processor properties: ${processorProperties}")
+
+ assert processorProperties.find { it.key == "Plaintext Property" }.value == "plain value"
+ assert processorProperties.find { it.key == "Sensitive Property" }.value == "sensitive value"
+ }
+
private
static Cipher generateCipher(boolean encryptMode, String password = DEFAULT_PASSWORD, byte[] salt = DEFAULT_SALT, int iterationCount = DEFAULT_ITERATION_COUNT) {
// Initialize secret key from password
http://git-wip-us.apache.org/repos/asf/nifi/blob/f6b171d5/nifi-nar-bundles/nifi-social-media-bundle/nifi-twitter-processors/src/main/java/org/apache/nifi/processors/twitter/GetTwitter.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-social-media-bundle/nifi-twitter-processors/src/main/java/org/apache/nifi/processors/twitter/GetTwitter.java b/nifi-nar-bundles/nifi-social-media-bundle/nifi-twitter-processors/src/main/java/org/apache/nifi/processors/twitter/GetTwitter.java
index e099449..f0a8b0c 100644
--- a/nifi-nar-bundles/nifi-social-media-bundle/nifi-twitter-processors/src/main/java/org/apache/nifi/processors/twitter/GetTwitter.java
+++ b/nifi-nar-bundles/nifi-social-media-bundle/nifi-twitter-processors/src/main/java/org/apache/nifi/processors/twitter/GetTwitter.java
@@ -16,9 +16,21 @@
*/
package org.apache.nifi.processors.twitter;
+import com.twitter.hbc.ClientBuilder;
+import com.twitter.hbc.core.Client;
+import com.twitter.hbc.core.Constants;
+import com.twitter.hbc.core.endpoint.Location;
+import com.twitter.hbc.core.endpoint.Location.Coordinate;
+import com.twitter.hbc.core.endpoint.StatusesFilterEndpoint;
+import com.twitter.hbc.core.endpoint.StatusesFirehoseEndpoint;
+import com.twitter.hbc.core.endpoint.StatusesSampleEndpoint;
+import com.twitter.hbc.core.endpoint.StreamingEndpoint;
+import com.twitter.hbc.core.event.Event;
+import com.twitter.hbc.core.processor.StringDelimitedProcessor;
+import com.twitter.hbc.httpclient.auth.Authentication;
+import com.twitter.hbc.httpclient.auth.OAuth1;
import java.io.IOException;
import java.io.OutputStream;
-import java.net.MalformedURLException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
@@ -31,7 +43,6 @@ import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.regex.Pattern;
-
import org.apache.nifi.annotation.behavior.InputRequirement;
import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
import org.apache.nifi.annotation.behavior.SupportsBatching;
@@ -56,24 +67,10 @@ import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.io.OutputStreamCallback;
import org.apache.nifi.processor.util.StandardValidators;
-import com.twitter.hbc.ClientBuilder;
-import com.twitter.hbc.core.Client;
-import com.twitter.hbc.core.Constants;
-import com.twitter.hbc.core.endpoint.Location ;
-import com.twitter.hbc.core.endpoint.Location.Coordinate ;
-import com.twitter.hbc.core.endpoint.StatusesFilterEndpoint;
-import com.twitter.hbc.core.endpoint.StatusesFirehoseEndpoint;
-import com.twitter.hbc.core.endpoint.StatusesSampleEndpoint;
-import com.twitter.hbc.core.endpoint.StreamingEndpoint;
-import com.twitter.hbc.core.event.Event;
-import com.twitter.hbc.core.processor.StringDelimitedProcessor;
-import com.twitter.hbc.httpclient.auth.Authentication;
-import com.twitter.hbc.httpclient.auth.OAuth1;
-
@SupportsBatching
@InputRequirement(Requirement.INPUT_FORBIDDEN)
@Tags({"twitter", "tweets", "social media", "status", "json"})
-@CapabilityDescription("Pulls status changes from Twitter's streaming API")
+@CapabilityDescription("Pulls status changes from Twitter's streaming API. In versions starting with 1.9.0, the Consumer Key and Access Token are marked as sensitive according to https://developer.twitter.com/en/docs/basics/authentication/guides/securing-keys-and-tokens")
@WritesAttribute(attribute = "mime.type", description = "Sets mime type to application/json")
public class GetTwitter extends AbstractProcessor {
@@ -92,6 +89,7 @@ public class GetTwitter extends AbstractProcessor {
.name("Consumer Key")
.description("The Consumer Key provided by Twitter")
.required(true)
+ .sensitive(true)
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.build();
public static final PropertyDescriptor CONSUMER_SECRET = new PropertyDescriptor.Builder()
@@ -105,6 +103,7 @@ public class GetTwitter extends AbstractProcessor {
.name("Access Token")
.description("The Access Token provided by Twitter")
.required(true)
+ .sensitive(true)
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.build();
public static final PropertyDescriptor ACCESS_TOKEN_SECRET = new PropertyDescriptor.Builder()
@@ -221,7 +220,7 @@ public class GetTwitter extends AbstractProcessor {
}
@OnScheduled
- public void onScheduled(final ProcessContext context) throws MalformedURLException {
+ public void onScheduled(final ProcessContext context) {
final String endpointName = context.getProperty(ENDPOINT).getValue();
final Authentication oauth = new OAuth1(context.getProperty(CONSUMER_KEY).getValue(),
context.getProperty(CONSUMER_SECRET).getValue(),
@@ -369,7 +368,7 @@ public class GetTwitter extends AbstractProcessor {
flowFile = session.putAllAttributes(flowFile, attributes);
session.transfer(flowFile, REL_SUCCESS);
- session.getProvenanceReporter().receive(flowFile, Constants.STREAM_HOST + client.getEndpoint().getURI().toString());
+ session.getProvenanceReporter().receive(flowFile, Constants.STREAM_HOST + client.getEndpoint().getURI());
}
private static class FollowingValidator implements Validator {