You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by al...@apache.org on 2020/01/13 20:16:13 UTC
[nifi-minifi] branch master updated: MINIFI-516: Added bootstrap
option to override processor SSLs with parent SSL context
This is an automated email from the ASF dual-hosted git repository.
aldrin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nifi-minifi.git
The following commit(s) were added to refs/heads/master by this push:
new 4e60ec2 MINIFI-516: Added bootstrap option to override processor SSLs with parent SSL context
4e60ec2 is described below
commit 4e60ec2585cd6813ac5b0ba2d89fcae135e6b37d
Author: r65535 <r6...@outlook.com>
AuthorDate: Thu Oct 10 08:56:39 2019 +0100
MINIFI-516: Added bootstrap option to override processor SSLs with parent SSL context
This closes #177.
Signed-off-by: Aldrin Piri <al...@apache.org>
---
.../bootstrap/util/BootstrapTransformer.java | 11 +++
.../minifi/bootstrap/util/ConfigTransformer.java | 14 +++
.../bootstrap/util/ConfigTransformerTest.java | 57 ++++++++++++
.../src/test/resources/MINIFI-516}/bootstrap.conf | 11 ++-
.../src/test/resources/MINIFI-516/config.yml | 102 +++++++++++++++++++++
.../schema/common/BootstrapPropertyKeys.java | 2 +
.../src/main/markdown/System_Admin_Guide.md | 8 ++
.../src/main/resources/conf/bootstrap.conf | 3 +
8 files changed, 204 insertions(+), 4 deletions(-)
diff --git a/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/BootstrapTransformer.java b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/BootstrapTransformer.java
index dc57990..eb861b7 100644
--- a/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/BootstrapTransformer.java
+++ b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/BootstrapTransformer.java
@@ -31,6 +31,7 @@ import static org.apache.nifi.minifi.commons.schema.common.BootstrapPropertyKeys
import static org.apache.nifi.minifi.commons.schema.common.BootstrapPropertyKeys.BOOTSTRAP_PROVENANCE_REPORTING_KEYS;
import static org.apache.nifi.minifi.commons.schema.common.BootstrapPropertyKeys.BOOTSTRAP_SECURITY_PROPERTY_KEYS;
import static org.apache.nifi.minifi.commons.schema.common.BootstrapPropertyKeys.BOOTSTRAP_SENSITIVE_PROPERTY_KEYS;
+import static org.apache.nifi.minifi.commons.schema.common.BootstrapPropertyKeys.USE_PARENT_SSL;
public class BootstrapTransformer {
@@ -88,4 +89,14 @@ public class BootstrapTransformer {
return provenanceReportingPropsOptional;
}
+ public static boolean processorSSLOverride(final Properties bootstrapProperties) {
+ boolean shouldOverride = false;
+
+ if (bootstrapProperties != null) {
+ shouldOverride = Boolean.parseBoolean(bootstrapProperties.getProperty(USE_PARENT_SSL));
+ }
+
+ return shouldOverride;
+ }
+
}
diff --git a/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java
index 0e92ece..da4af51 100644
--- a/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java
+++ b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java
@@ -45,6 +45,8 @@ import org.apache.nifi.minifi.commons.schema.common.ConvertableSchema;
import org.apache.nifi.minifi.commons.schema.common.Schema;
import org.apache.nifi.minifi.commons.schema.common.StringUtil;
import org.apache.nifi.minifi.commons.schema.serialization.SchemaLoader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.w3c.dom.DOMException;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
@@ -82,6 +84,8 @@ public final class ConfigTransformer {
public static final String DEFAULT_PROV_REPORTING_TASK_CLASS = "org.apache.nifi.reporting.SiteToSiteProvenanceReportingTask";
public static final String NIFI_VERSION_KEY = "nifi.version";
+ public static final Logger logger = LoggerFactory.getLogger(ConfigTransformer.class);
+
// Final util classes should have private constructor
private ConfigTransformer() {
}
@@ -96,9 +100,19 @@ public final class ConfigTransformer {
// See if we are providing defined properties from the filesystem configurations and use those as the definitive values
if (securityProperties != null) {
configSchema.setSecurityProperties(securityProperties);
+ logger.info("Bootstrap flow override: Replaced security properties");
}
if (provenanceReportingProperties != null) {
configSchema.setProvenanceReportingProperties(provenanceReportingProperties);
+ logger.info("Bootstrap flow override: Replaced provenance reporting properties");
+ }
+
+ // Replace all processor SSL controller services with MiNiFi parent, if bootstrap boolean is set to true
+ if (BootstrapTransformer.processorSSLOverride(bootstrapProperties)) {
+ for (ProcessorSchema processorConfig : configSchema.getProcessGroupSchema().getProcessors()) {
+ processorConfig.getProperties().replace("SSL Context Service", processorConfig.getProperties().get("SSL Context Service"), "SSL-Context-Service");
+ logger.info("Bootstrap flow override: Replaced {} SSL Context Service with parent MiNiFi SSL", processorConfig.getName());
+ }
}
// Create nifi.properties and flow.xml.gz in memory
diff --git a/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java b/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java
index 5c36cee..32a5852 100644
--- a/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java
+++ b/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java
@@ -53,6 +53,7 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.io.StringBufferInputStream;
import java.nio.charset.Charset;
import java.util.Arrays;
import java.util.Collections;
@@ -64,7 +65,9 @@ import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.stream.Collectors;
+import java.util.zip.GZIPInputStream;
+import static org.apache.nifi.minifi.bootstrap.RunMiNiFiTest.getTestBootstrapProperties;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
@@ -463,6 +466,39 @@ public class ConfigTransformerTest {
}
}
+ @Test
+ public void checkSSLOverrides() throws Exception {
+ File inputFile = new File("./src/test/resources/MINIFI-516/config.yml");
+ final Properties bootstrapProperties = getTestBootstrapProperties("MINIFI-516/bootstrap.conf");
+ ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", bootstrapProperties);
+
+ // nifi.properties testing
+ File nifiPropertiesFile = new File("./target/nifi.properties");
+ assertTrue(nifiPropertiesFile.exists());
+ assertTrue(nifiPropertiesFile.canRead());
+
+ nifiPropertiesFile.deleteOnExit();
+
+ // flow.xml.gz testing
+ File flowXml = new File("./target/flow.xml.gz");
+ assertTrue(flowXml.exists());
+ assertTrue(flowXml.canRead());
+
+ String flow = loadFlowXML(new FileInputStream(flowXml));
+
+ DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ DocumentBuilder db = dbf.newDocumentBuilder();
+ Document xml = db.parse(new StringBufferInputStream(flow));
+
+ XPath xPath = XPathFactory.newInstance().newXPath();
+ String result = xPath.evaluate("/flowController/rootGroup/processor/property[name = \"SSL Context Service\"]/value/text()", xml);
+
+ assertEquals(result, "SSL-Context-Service");
+
+ flowXml.deleteOnExit();
+
+ }
+
public void testConfigFileTransform(String configFile) throws Exception {
ConfigSchema configSchema = SchemaLoader.loadConfigSchemaFromYaml(ConfigTransformerTest.class.getClassLoader().getResourceAsStream(configFile));
@@ -664,4 +700,25 @@ public class ConfigTransformerTest {
}
}
}
+
+ public static Properties getTestBootstrapProperties(final String fileName) throws IOException {
+ final Properties bootstrapProperties = new Properties();
+ try (final InputStream fis = ConfigTransformerTest.class.getClassLoader().getResourceAsStream(fileName)) {
+ bootstrapProperties.load(fis);
+ }
+ return bootstrapProperties;
+ }
+
+ public static String loadFlowXML(InputStream compressedData) throws IOException {
+ ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+ GZIPInputStream gzipInputStream = new GZIPInputStream(compressedData);
+
+ byte[] buffer = new byte[1024];
+ int len;
+ while ((len = gzipInputStream.read(buffer)) != -1) {
+ byteArrayOutputStream.write(buffer, 0, len);
+ }
+
+ return byteArrayOutputStream.toString();
+ }
}
diff --git a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/bootstrap.conf b/minifi-bootstrap/src/test/resources/MINIFI-516/bootstrap.conf
similarity index 96%
copy from minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/bootstrap.conf
copy to minifi-bootstrap/src/test/resources/MINIFI-516/bootstrap.conf
index c8c40cf..2fe540d 100644
--- a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/bootstrap.conf
+++ b/minifi-bootstrap/src/test/resources/MINIFI-516/bootstrap.conf
@@ -19,7 +19,7 @@
java=java
# Username to use when running MiNiFi. This value will be ignored on Windows.
-run.as=${minifi.run.as}
+run.as=
# Configure where MiNiFi's lib and conf directories live
# When running as a Windows service set full paths instead of relative paths
@@ -59,6 +59,9 @@ nifi.minifi.provenance.reporting.instance.url=
nifi.minifi.provenance.reporting.batch.size=
nifi.minifi.provenance.reporting.communications.timeout=
+# Ignore custom SSL controller services and use parent minifi SSL
+nifi.minifi.flow.use.parent.ssl=true
+
# Notifiers to use for the associated agent, comma separated list of class names
#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.FileChangeIngestor
#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.RestChangeIngestor
@@ -105,8 +108,8 @@ nifi.minifi.provenance.reporting.communications.timeout=
java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true
# JVM memory settings
-java.arg.2=-Xms${minifi.jvm.heap.mb}m
-java.arg.3=-Xmx${minifi.jvm.heap.mb}m
+java.arg.2=-Xms256m
+java.arg.3=-Xmx256m
# Enable Remote Debugging
#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000
@@ -126,4 +129,4 @@ java.arg.7=-Djava.security.egd=file:/dev/urandom
#java.arg.13=-XX:+UseG1GC
#Set headless mode by default
-java.arg.14=-Djava.awt.headless=true
+java.arg.14=-Djava.awt.headless=true
\ No newline at end of file
diff --git a/minifi-bootstrap/src/test/resources/MINIFI-516/config.yml b/minifi-bootstrap/src/test/resources/MINIFI-516/config.yml
new file mode 100644
index 0000000..933e124
--- /dev/null
+++ b/minifi-bootstrap/src/test/resources/MINIFI-516/config.yml
@@ -0,0 +1,102 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the \"License\"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an \"AS IS\" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+MiNiFi Config Version: 3
+Flow Controller:
+ name: listenhttp-withssl
+ comment: ''
+Core Properties:
+ flow controller graceful shutdown period: 10 sec
+ flow service write delay interval: 500 ms
+ administrative yield duration: 30 sec
+ bored yield duration: 10 millis
+ max concurrent threads: 1
+ variable registry properties: ''
+FlowFile Repository:
+ partitions: 256
+ checkpoint interval: 2 mins
+ always sync: false
+ Swap:
+ threshold: 20000
+ in period: 5 sec
+ in threads: 1
+ out period: 5 sec
+ out threads: 4
+Content Repository:
+ content claim max appendable size: 10 MB
+ content claim max flow files: 100
+ always sync: false
+Provenance Repository:
+ provenance rollover time: 1 min
+ implementation: org.apache.nifi.provenance.MiNiFiPersistentProvenanceRepository
+Component Status Repository:
+ buffer size: 1440
+ snapshot frequency: 1 min
+Security Properties:
+ keystore: ''
+ keystore type: ''
+ keystore password: ''
+ key password: ''
+ truststore: ''
+ truststore type: ''
+ truststore password: ''
+ ssl protocol: ''
+ Sensitive Props:
+ key:
+ algorithm: PBEWITHMD5AND256BITAES-CBC-OPENSSL
+ provider: BC
+Processors:
+- id: d636b1bb-fdc7-3e7e-0000-000000000000
+ name: ListenHTTP
+ class: org.apache.nifi.processors.standard.ListenHTTP
+ max concurrent tasks: 1
+ scheduling strategy: TIMER_DRIVEN
+ scheduling period: 0 sec
+ penalization period: 30 sec
+ yield period: 1 sec
+ run duration nanos: 0
+ auto-terminated relationships list:
+ - success
+ Properties:
+ Authorized DN Pattern: .*
+ Base Path: contentListener
+ HTTP Headers to receive as Attributes (Regex): .*
+ Listening Port: '11223'
+ Max Data to Receive per Second:
+ Max Unconfirmed Flowfile Time: 60 secs
+ Return Code: '200'
+ SSL Context Service: c6e0b2ac-9fa8-3e31-0000-000000000000
+ multipart-read-buffer-size: 512 KB
+ multipart-request-max-size: 1 MB
+Controller Services:
+- id: c6e0b2ac-9fa8-3e31-0000-000000000000
+ name: CustomSSL
+ type: org.apache.nifi.ssl.StandardRestrictedSSLContextService
+ Properties:
+ Keystore Filename: /tmp/keystore.jks
+ Keystore Password:
+ Keystore Type: JKS
+ SSL Protocol:
+ Truststore Filename: /tmp/truststore.jks
+ Truststore Password:
+ Truststore Type: JKS
+ key-password:
+Process Groups: []
+Input Ports: []
+Output Ports: []
+Funnels: []
+Connections: []
+Remote Process Groups: []
+NiFi Properties Overrides: {}
\ No newline at end of file
diff --git a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/BootstrapPropertyKeys.java b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/BootstrapPropertyKeys.java
index 5d7724e..972dd13 100644
--- a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/BootstrapPropertyKeys.java
+++ b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/BootstrapPropertyKeys.java
@@ -42,6 +42,8 @@ public class BootstrapPropertyKeys {
public static final String STATUS_REPORTER_PROPERTY_PREFIX = "nifi.minifi.status.reporter";
public static final String STATUS_REPORTER_COMPONENTS_KEY = STATUS_REPORTER_PROPERTY_PREFIX + ".components";
+ public static final String USE_PARENT_SSL = "nifi.minifi.flow.use.parent.ssl";
+
public static final String SECURITY_KEYSTORE_KEY = "nifi.minifi.security.keystore";
public static final String SECURITY_KEYSTORE_TYPE_KEY = "nifi.minifi.security.keystoreType";
public static final String SECURITY_KEYSTORE_PASSWORD_KEY = "nifi.minifi.security.keystorePasswd";
diff --git a/minifi-docs/src/main/markdown/System_Admin_Guide.md b/minifi-docs/src/main/markdown/System_Admin_Guide.md
index b554615..b2e4b18 100644
--- a/minifi-docs/src/main/markdown/System_Admin_Guide.md
+++ b/minifi-docs/src/main/markdown/System_Admin_Guide.md
@@ -694,6 +694,14 @@ You can now install the MiNiFi service by running the `install-service.bat` scri
The *minifi.exe* in MiNiFi `bin` directory is used to run MiNiFi Windows service. The bundled one is for 64 bit architecture and requires 64 bit JRE. If you have to use 32 bit JRE for some reason, you need to replace the *minifi.exe* file with the one for 32 bit to make MiNiFi service runs successfully. To do so, go to [Commons Daemon project download page](https://commons.apache.org/proper/commons-daemon/download_daemon.cgi), download the binary (e.g. _commons-daemon-1.1.0-bin.zip_), ex [...]
+# Flow overriding with bootstrap.conf
+
+ The following options can be set to override flow properties in the config.yml
+
+ *bootstrap.conf Property* | *Description*
+ ----------------------------------|--------------------
+ `nifi.minifi.flow.use.parent.ssl` | When set to true, all processors will reference the MiNiFi parent SSL controller service defined in the security properties, instead of custom controller services.
+
# Example Config File
Below are two example config YAML files. The first tails the *minifi-app.log* and sends the tailed log and provenance data back to a secure instance of NiFi. The second uses a series of processors to tail the app log, routes off only lines that contain "WriteAheadFlowFileRepository" and puts it as a file in the "./" directory.
diff --git a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/bootstrap.conf b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/bootstrap.conf
index c8c40cf..74006b4 100644
--- a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/bootstrap.conf
+++ b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/bootstrap.conf
@@ -59,6 +59,9 @@ nifi.minifi.provenance.reporting.instance.url=
nifi.minifi.provenance.reporting.batch.size=
nifi.minifi.provenance.reporting.communications.timeout=
+# Ignore all processor SSL controller services and use parent minifi SSL instead
+ nifi.minifi.flow.use.parent.ssl=false
+
# Notifiers to use for the associated agent, comma separated list of class names
#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.FileChangeIngestor
#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.RestChangeIngestor