You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by jo...@apache.org on 2020/02/21 21:18:44 UTC
[nifi] branch support/nifi-1.11.x updated (a38cf14 -> 9748695)
This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a change to branch support/nifi-1.11.x
in repository https://gitbox.apache.org/repos/asf/nifi.git.
from a38cf14 NIFI-7155 updated dockerfile versions for next release
new 5ed7dbc NIFI-7135 - Fix Java 11 build with com.puppycrawl.tools:checkstyle:jar:8.29 dependency
new 0466918 NIFI-7175 removed appveyor and travis builds which have become unreliable. Added Github Actions based CI build for pull requests and pushes. Removed unit tests which make unreadable or unaccesible dirs. Includes windows build as well as ubuntu. we can do localization based builds later as this is all proven stable.
new 6504829 NIFI-7175 README.md
new fc89f98 NIFI-7049 : SFTP processors shouldn't silently try to access known hosts file of the user
new 7e74984 NIFI-7109 Unit tests should be able to determine if item validator was called - Create a mock Validator to track count of calls to validate(). We cannot use Mockito for this, because it can't mock all the StandardValidators
new c276ac5 NIFI-7136 Added autocomplete="off" to login password input (#4055)
new 1c6863a NIFI-7143: Upgrade GCP dependency
new 84c28cf NIFI-6064 MockComponentLog misplaces reported exceptions
new 1a07d26 NIFI-7142: Automatically handle schema drift in the PutKudu processor
new e4b905b NIFI-6867: Validate PutKudu operation type property
new 6b11736 NIFI-6551: Improve PutKudu timestamp handling
new 46d3b6b NIFI-6927 Consolidate SSL context and trust managers for OkHttp on JDK9. Fixes name conflicts.
new ea13898 NIFI-7165 Fixed TLS Toolkit Guide flags with default validity days.
new 9748695 NIFI-7175 Fixed core attributes formatting in Developer Guide. (#4066)
The 14 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/ci-workflow.yml | 111 ++++++++++++++++++
.travis-output-filters | 15 ---
.travis.sh | 28 -----
.travis.yml | 126 --------------------
README.md | 2 +-
appveyor.yml | 38 ------
.../notification/http/HttpNotificationService.java | 69 +++--------
.../nifi/security/util/SslContextFactory.java | 71 +++++++++---
.../validator/InstrumentedStandardValidator.java | 92 +++++++++++++++
.../util/validator/TestStandardValidators.java | 73 +++++++++---
nifi-docs/src/main/asciidoc/developer-guide.adoc | 30 ++---
nifi-docs/src/main/asciidoc/toolkit-guide.adoc | 4 +-
.../java/org/apache/nifi/util/CapturingLogger.java | 5 +
.../nifi/util/validator/InstrumentedValidator.java | 92 +++++++++++++++
.../nifi/util/TestStandardProcessorTestRunner.java | 19 +++
.../nifi-elasticsearch-processors/pom.xml | 2 +-
.../AbstractElasticsearchHttpProcessor.java | 31 ++++-
.../TestQueryElasticsearchHttpNoHits.java | 2 +-
.../okhttp/OkHttpReplicationClient.java | 84 +++++---------
.../security/util/SslServerSocketFactory.java | 82 -------------
.../framework/security/util/SslSocketFactory.java | 92 ---------------
.../webapp/WEB-INF/partials/login/login-form.jsp | 4 +-
.../nifi-gcp-bundle/nifi-gcp-services-api/pom.xml | 2 +-
nifi-nar-bundles/nifi-gcp-bundle/pom.xml | 2 +-
.../processors/kudu/AbstractKuduProcessor.java | 81 +++++++++----
.../org/apache/nifi/processors/kudu/PutKudu.java | 110 ++++++++++++++++--
.../org/apache/nifi/processors/kudu/ITPutKudu.java | 42 +++++--
.../apache/nifi/processors/kudu/TestPutKudu.java | 5 +-
.../nifi/processors/standard/InvokeHTTP.java | 20 +++-
.../processors/standard/util/SFTPTransfer.java | 20 ++--
.../nifi/processors/standard/TestGetFile.java | 127 ++-------------------
.../nifi/processors/standard/TestGetSFTP.java | 29 +++++
.../org/apache/nifi/lookup/RestLookupService.java | 38 +++++-
pom.xml | 29 ++++-
34 files changed, 860 insertions(+), 717 deletions(-)
create mode 100644 .github/workflows/ci-workflow.yml
delete mode 100644 .travis-output-filters
delete mode 100755 .travis.sh
delete mode 100644 .travis.yml
delete mode 100644 appveyor.yml
create mode 100644 nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/validator/InstrumentedStandardValidator.java
create mode 100644 nifi-mock/src/main/java/org/apache/nifi/util/validator/InstrumentedValidator.java
delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-security/src/main/java/org/apache/nifi/framework/security/util/SslServerSocketFactory.java
delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-security/src/main/java/org/apache/nifi/framework/security/util/SslSocketFactory.java
[nifi] 03/14: NIFI-7175 README.md
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a commit to branch support/nifi-1.11.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit 65048299c8e5890d32f002b18f64b9060e283de2
Author: Pierre Villard <pi...@gmail.com>
AuthorDate: Thu Feb 20 16:41:40 2020 -0800
NIFI-7175 README.md
---
README.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/README.md b/README.md
index 72ca973..762dfd2 100644
--- a/README.md
+++ b/README.md
@@ -14,7 +14,7 @@
-->
[<img src="https://nifi.apache.org/assets/images/apache-nifi-logo.svg" width="300" height="126" alt="Apache NiFi"/>][nifi]
-[
+[](https://github.com/apache/nifi/actions)
[](https://hub.docker.com/r/apache/nifi/)
[](https://nifi.apache.org/download.html)
[](https://s.apache.org/nifi-community-slack)
[nifi] 08/14: NIFI-6064 MockComponentLog misplaces reported
exceptions
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a commit to branch support/nifi-1.11.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit 84c28cff6e209a8dfb611f9d2ae42ba67fd99da3
Author: karthik-kadajji <ka...@ultratendency.com>
AuthorDate: Tue Feb 18 18:49:44 2020 +0100
NIFI-6064 MockComponentLog misplaces reported exceptions
Signed-off-by: Pierre Villard <pi...@gmail.com>
This closes #4061.
---
.../java/org/apache/nifi/util/CapturingLogger.java | 5 +++++
.../nifi/util/TestStandardProcessorTestRunner.java | 19 +++++++++++++++++++
2 files changed, 24 insertions(+)
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/CapturingLogger.java b/nifi-mock/src/main/java/org/apache/nifi/util/CapturingLogger.java
index 25b37a0..a289eaa 100644
--- a/nifi-mock/src/main/java/org/apache/nifi/util/CapturingLogger.java
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/CapturingLogger.java
@@ -372,6 +372,11 @@ public class CapturingLogger implements Logger {
logger.error(format, arg1, arg2);
}
+ public void error(String format, Object arg1, Throwable t) {
+ errorMessages.add(new LogMessage(null, format, t, arg1));
+ logger.error(format, arg1, t);
+ }
+
@Override
public void error(String format, Object... arguments) {
errorMessages.add(new LogMessage(null, format, null, arguments));
diff --git a/nifi-mock/src/test/java/org/apache/nifi/util/TestStandardProcessorTestRunner.java b/nifi-mock/src/test/java/org/apache/nifi/util/TestStandardProcessorTestRunner.java
index 134d2b8..f2febb4 100644
--- a/nifi-mock/src/test/java/org/apache/nifi/util/TestStandardProcessorTestRunner.java
+++ b/nifi-mock/src/test/java/org/apache/nifi/util/TestStandardProcessorTestRunner.java
@@ -381,4 +381,23 @@ public class TestStandardProcessorTestRunner {
return Arrays.asList(namePropertyDescriptor);
}
}
+
+ @Test
+ public void testErrorLogMessageArguments() {
+
+ String compName = "name of component";
+ final MockComponentLog logger = new MockComponentLog("first id",compName);
+
+ Throwable t = new ArithmeticException();
+ logger.error("expected test error",t);
+
+ String expected_throwable = "java.lang.ArithmeticException";
+
+ List<LogMessage> log = logger.getErrorMessages();
+ LogMessage msg = log.get(0);
+ // checking if the error messages are recorded in the correct throwable argument.
+ assertEquals(expected_throwable,msg.getThrowable().toString());
+ assertEquals("{} expected test error",msg.getMsg());
+
+ }
}
[nifi] 04/14: NIFI-7049 : SFTP processors shouldn't silently try to
access known hosts file of the user
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a commit to branch support/nifi-1.11.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit fc89f9861d7e7988737ddad9d1646f67391402d3
Author: mdayakar <da...@gmail.com>
AuthorDate: Thu Jan 23 23:45:21 2020 +0530
NIFI-7049 : SFTP processors shouldn't silently try to access known hosts file of the user
Signed-off-by: Arpad Boda <ab...@apache.org>
This closes #4014
---
.../processors/standard/util/SFTPTransfer.java | 20 +++++++++------
.../nifi/processors/standard/TestGetSFTP.java | 29 ++++++++++++++++++++++
2 files changed, 41 insertions(+), 8 deletions(-)
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/SFTPTransfer.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/SFTPTransfer.java
index dda1456..465bdde 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/SFTPTransfer.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/SFTPTransfer.java
@@ -94,7 +94,10 @@ public class SFTPTransfer implements FileTransfer {
.build();
public static final PropertyDescriptor HOST_KEY_FILE = new PropertyDescriptor.Builder()
.name("Host Key File")
- .description("If supplied, the given file will be used as the Host Key; otherwise, no use host key file will be used")
+ .description("If supplied, the given file will be used as the Host Key;" +
+ " otherwise, if 'Strict Host Key Checking' property is applied (set to true)" +
+ " then uses the 'known_hosts' and 'known_hosts2' files from ~/.ssh directory" +
+ " else no host key file will be used")
.addValidator(StandardValidators.FILE_EXISTS_VALIDATOR)
.required(false)
.build();
@@ -548,20 +551,21 @@ public class SFTPTransfer implements FileTransfer {
});
}
+ // If strict host key checking is false, add a HostKeyVerifier that always returns true
+ final boolean strictHostKeyChecking = ctx.getProperty(STRICT_HOST_KEY_CHECKING).asBoolean();
+ if (!strictHostKeyChecking) {
+ sshClient.addHostKeyVerifier(new PromiscuousVerifier());
+ }
+
// Load known hosts file if specified, otherwise load default
final String hostKeyVal = ctx.getProperty(HOST_KEY_FILE).getValue();
if (hostKeyVal != null) {
sshClient.loadKnownHosts(new File(hostKeyVal));
- } else {
+ // Load default known_hosts file only when 'Strict Host Key Checking' property is enabled
+ } else if (strictHostKeyChecking) {
sshClient.loadKnownHosts();
}
- // If strict host key checking is false, add a HostKeyVerifier that always returns true
- final boolean strictHostKeyChecking = ctx.getProperty(STRICT_HOST_KEY_CHECKING).asBoolean();
- if (!strictHostKeyChecking) {
- sshClient.addHostKeyVerifier(new PromiscuousVerifier());
- }
-
// Enable compression on the client if specified in properties
final PropertyValue compressionValue = ctx.getProperty(FileTransfer.USE_COMPRESSION);
if (compressionValue != null && "true".equalsIgnoreCase(compressionValue.getValue())) {
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGetSFTP.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGetSFTP.java
index a4f532a..5d063c7 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGetSFTP.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGetSFTP.java
@@ -97,6 +97,35 @@ public class TestGetSFTP {
}
@Test
+ public void testGetSFTPShouldNotThrowIOExceptionIfUserHomeDirNotExixts() throws IOException {
+ emptyTestDirectory();
+
+ String userHome = System.getProperty("user.home");
+ try {
+ // Set 'user.home' system property value to not_existdir
+ System.setProperty("user.home", "/not_existdir");
+ touchFile(sshTestServer.getVirtualFileSystemPath() + "testFile1.txt");
+ touchFile(sshTestServer.getVirtualFileSystemPath() + "testFile2.txt");
+
+ getSFTPRunner.run();
+
+ getSFTPRunner.assertTransferCount(GetSFTP.REL_SUCCESS, 2);
+
+ // Verify files deleted
+ for (int i = 1; i < 3; i++) {
+ Path file1 = Paths.get(sshTestServer.getVirtualFileSystemPath() + "/testFile" + i + ".txt");
+ Assert.assertTrue("File not deleted.", !file1.toAbsolutePath().toFile().exists());
+ }
+
+ getSFTPRunner.clearTransferState();
+
+ } finally {
+ // set back the original value for 'user.home' system property
+ System.setProperty("user.home", userHome);
+ }
+ }
+
+ @Test
public void testGetSFTPIgnoreDottedFiles() throws IOException {
emptyTestDirectory();
[nifi] 05/14: NIFI-7109 Unit tests should be able to determine if
item validator was called - Create a mock Validator to track count of calls
to validate(). We cannot use Mockito for this,
because it can't mock all the StandardValidators
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a commit to branch support/nifi-1.11.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit 7e74984e1df62ced5f40e202f75d8f045f6d4d6e
Author: Otto Fowler <ot...@gmail.com>
AuthorDate: Thu Feb 6 10:07:49 2020 -0500
NIFI-7109 Unit tests should be able to determine if item validator was called
- Create a mock Validator to track count of calls to validate().
We cannot use Mockito for this, because it can't mock all the
StandardValidators
refactor based on review comments
fix naming in comments
moved to main based on review
Signed-off-by: Matthew Burgess <ma...@apache.org>
This closes #4043
---
.../validator/InstrumentedStandardValidator.java | 92 ++++++++++++++++++++++
.../util/validator/TestStandardValidators.java | 73 +++++++++++++----
.../nifi/util/validator/InstrumentedValidator.java | 92 ++++++++++++++++++++++
3 files changed, 243 insertions(+), 14 deletions(-)
diff --git a/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/validator/InstrumentedStandardValidator.java b/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/validator/InstrumentedStandardValidator.java
new file mode 100644
index 0000000..b5f6f39
--- /dev/null
+++ b/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/validator/InstrumentedStandardValidator.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.validator;
+
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.components.Validator;
+
+/**:
+ * InstrumentedStandarValidator wraps a {@class Validator} and provides statistics on it's interactions.
+ * Because many of the {@class Validator} instances returned from {@class StandardValidator }
+ * are not mockable with with mockito, this is required to know, when running a test, if a
+ * {@class Validator} was in fact called, for example.
+ */
+public class InstrumentedStandardValidator implements Validator {
+
+ /**
+ * Flag to reset a count after retrieving it.
+ * Thus not having to explicitly call reset() for simple cases.
+ */
+ boolean doReset = false;
+
+ /**
+ * Count the number of calls to validate()
+ */
+ private int validateCallCount;
+ private Validator mockedValidator;
+
+ /**
+ * Constructs a new {@class InstrumentedStandarValidator}.
+ *
+ * @param mockedValidator the {@class Validator} to wrap.
+ */
+ public InstrumentedStandardValidator(Validator mockedValidator) {
+ this(mockedValidator,false);
+ }
+
+ /**
+ * Constructs a new {@class InstrumentedStandarValidator}.
+ *
+ * @param mockedValidator the {@class Validator} to wrap.
+ */
+ public InstrumentedStandardValidator(Validator mockedValidator, boolean resetOnGet) {
+ this.mockedValidator = mockedValidator;
+ this.doReset = resetOnGet;
+ }
+
+ /**
+ * Default constructor without wrapping not supported.
+ *
+ */
+ private InstrumentedStandardValidator(){}
+
+ @Override
+ public ValidationResult validate(String subject, String input, ValidationContext context) {
+ validateCallCount++;
+ return mockedValidator.validate(subject, input, context);
+ }
+
+ /**
+ * Returns the count of times validate was called
+ * @return count of validate() calls
+ */
+ public int getValidateCallCount() {
+ int count = validateCallCount;
+ if (doReset) {
+ validateCallCount = 0;
+ }
+ return count;
+ }
+
+ /**
+ * Resets the count of all calls to 0.
+ */
+ public void resetAll() {
+ validateCallCount = 0;
+ }
+}
diff --git a/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/validator/TestStandardValidators.java b/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/validator/TestStandardValidators.java
index f02946b..e469955 100644
--- a/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/validator/TestStandardValidators.java
+++ b/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/validator/TestStandardValidators.java
@@ -16,6 +16,13 @@
*/
package org.apache.nifi.util.validator;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+
+import java.util.concurrent.TimeUnit;
+
import org.apache.nifi.components.ValidationContext;
import org.apache.nifi.components.ValidationResult;
import org.apache.nifi.components.Validator;
@@ -23,12 +30,6 @@ import org.apache.nifi.processor.util.StandardValidators;
import org.junit.Test;
import org.mockito.Mockito;
-import java.util.concurrent.TimeUnit;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.mock;
-
public class TestStandardValidators {
@Test
@@ -102,7 +103,8 @@ public class TestStandardValidators {
@Test
public void testTimePeriodValidator() {
- Validator val = StandardValidators.createTimePeriodValidator(1L, TimeUnit.SECONDS, Long.MAX_VALUE, TimeUnit.NANOSECONDS);
+ Validator val = StandardValidators
+ .createTimePeriodValidator(1L, TimeUnit.SECONDS, Long.MAX_VALUE, TimeUnit.NANOSECONDS);
ValidationResult vr;
final ValidationContext validationContext = Mockito.mock(ValidationContext.class);
@@ -159,90 +161,133 @@ public class TestStandardValidators {
@Test
public void testListValidator() {
- Validator val = StandardValidators.createListValidator(true, false, StandardValidators.NON_EMPTY_VALIDATOR);
+ // use the TestMockValidator to be sure the item validator get's called when we think it should
+ // note that it will reset the count after every get call.
+ // note that we fail fast, so if there are 3 items, and the second fails validation, the call
+ // count will be 2
+ InstrumentedStandardValidator mockValidator = new InstrumentedStandardValidator(StandardValidators.NON_EMPTY_VALIDATOR, true);
+ Validator val = StandardValidators.createListValidator(true, false, mockValidator);
ValidationResult vr;
final ValidationContext validationContext = Mockito.mock(ValidationContext.class);
vr = val.validate("List", null, validationContext);
assertFalse(vr.isValid());
+ assertEquals(0, mockValidator.getValidateCallCount());
vr = val.validate("List", "", validationContext);
assertFalse(vr.isValid());
+ assertEquals(1, mockValidator.getValidateCallCount());
// Whitespace will be trimmed
vr = val.validate("List", " ", validationContext);
assertFalse(vr.isValid());
+ assertEquals(1, mockValidator.getValidateCallCount());
vr = val.validate("List", "1", validationContext);
assertTrue(vr.isValid());
+ assertEquals(1, mockValidator.getValidateCallCount());
vr = val.validate("List", "1,2,3", validationContext);
assertTrue(vr.isValid());
+ assertEquals(3, mockValidator.getValidateCallCount());
// The parser will not bother with whitespace after the last comma
vr = val.validate("List", "a,", validationContext);
assertTrue(vr.isValid());
+ assertEquals(1, mockValidator.getValidateCallCount());
// However it will bother if there is an empty element in the list (two commas in a row, e.g.)
vr = val.validate("List", "a,,c", validationContext);
assertFalse(vr.isValid());
+ assertEquals(2, mockValidator.getValidateCallCount());
vr = val.validate("List", "a, ,c, ", validationContext);
assertFalse(vr.isValid());
+ assertEquals(2, mockValidator.getValidateCallCount());
// Try without trim and use a non-blank validator instead of a non-empty one
- val = StandardValidators.createListValidator(false, true, StandardValidators.NON_BLANK_VALIDATOR);
+ //
+ // use the TestMockValidator to be sure the item validator get's called when we think it should
+ // note that it will reset the count after every get call.
+ // note that we fail fast, so if there are 3 items, and the second fails validation, the call
+ // count will be 2
+ mockValidator = new InstrumentedStandardValidator(StandardValidators.NON_BLANK_VALIDATOR, true);
+ val = StandardValidators.createListValidator(false, true, mockValidator);
vr = val.validate("List", null, validationContext);
assertFalse(vr.isValid());
+ assertEquals(0, mockValidator.getValidateCallCount());
- // Validator will ignore empty entries
+ // List Validator will ignore empty entries
vr = val.validate("List", "", validationContext);
assertTrue(vr.isValid());
+ assertEquals(0, mockValidator.getValidateCallCount());
// Whitespace will not be trimmed, but it is still invalid because a non-blank validator is used
vr = val.validate("List", " ", validationContext);
assertFalse(vr.isValid());
+ assertEquals(1, mockValidator.getValidateCallCount());
vr = val.validate("List", "a,,c", validationContext);
assertTrue(vr.isValid());
+ assertEquals(2, mockValidator.getValidateCallCount());
vr = val.validate("List", "a, ,c, ", validationContext);
assertFalse(vr.isValid());
+ assertEquals(2, mockValidator.getValidateCallCount());
// Try without trim and use a non-empty validator
- val = StandardValidators.createListValidator(false, false, StandardValidators.NON_EMPTY_VALIDATOR);
+ // use the TestMockValidator to be sure the item validator get's called when we think it should
+ // note that it will reset the count after every get call.
+ // note that we fail fast, so if there are 3 items, and the second fails validation, the call
+ // count will be 2
+ mockValidator = new InstrumentedStandardValidator(StandardValidators.NON_EMPTY_VALIDATOR, true);
+ val = StandardValidators.createListValidator(false, false, mockValidator);
vr = val.validate("List", null, validationContext);
assertFalse(vr.isValid());
+ assertEquals(0, mockValidator.getValidateCallCount());
vr = val.validate("List", "", validationContext);
assertFalse(vr.isValid());
+ assertEquals(1, mockValidator.getValidateCallCount());
// Whitespace will not be trimmed
vr = val.validate("List", " ", validationContext);
assertTrue(vr.isValid());
+ assertEquals(1, mockValidator.getValidateCallCount());
vr = val.validate("List", "a, ,c, ", validationContext);
assertTrue(vr.isValid());
-
- // Try with trim and use a boolean validator
- val = StandardValidators.createListValidator(true, true, StandardValidators.BOOLEAN_VALIDATOR);
+ assertEquals(4, mockValidator.getValidateCallCount());
+
+ // Try with trim and use a boolean validator // Try without trim and use a non-empty validator
+ // use the TestMockValidator to be sure the item validator get's called when we think it should
+ // note that it will reset the count after every get call.
+ // note that we fail fast, so if there are 3 items, and the second fails validation, the call
+ // count will be 2
+ mockValidator = new InstrumentedStandardValidator(StandardValidators.BOOLEAN_VALIDATOR, true);
+ val = StandardValidators.createListValidator(true, true, mockValidator);
vr = val.validate("List", "notbool", validationContext);
assertFalse(vr.isValid());
+ assertEquals(1, mockValidator.getValidateCallCount());
vr = val.validate("List", " notbool \n ", validationContext);
assertFalse(vr.isValid());
+ assertEquals(1, mockValidator.getValidateCallCount());
vr = val.validate("List", "true", validationContext);
assertTrue(vr.isValid());
+ assertEquals(1, mockValidator.getValidateCallCount());
vr = val.validate("List", " true \n ", validationContext);
assertTrue(vr.isValid());
+ assertEquals(1, mockValidator.getValidateCallCount());
vr = val.validate("List", " , false, true,\n", validationContext);
assertTrue(vr.isValid());
+ assertEquals(2, mockValidator.getValidateCallCount());
}
@Test
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/validator/InstrumentedValidator.java b/nifi-mock/src/main/java/org/apache/nifi/util/validator/InstrumentedValidator.java
new file mode 100644
index 0000000..77494fa
--- /dev/null
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/validator/InstrumentedValidator.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.validator;
+
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.components.Validator;
+
+/**
+ * InstrumentedValidator wraps a {@class Validator} and provides statistics on it's interactions.
+ * Because many of the {@class Validator} instances returned from {@class StandardValidator }
+ * are not mockable with with mockito, this is required to know, when running a test, if a
+ * {@class Validator} was in fact called, for example.
+ */
+public class InstrumentedValidator implements Validator {
+
+ /**
+ * Flag to reset a count after retrieving it.
+ * Thus not having to explicitly call reset() for simple cases.
+ */
+ boolean doReset = false;
+
+ /**
+ * Count the number of calls to validate()
+ */
+ private int validateCallCount;
+ private Validator mockedValidator;
+
+ /**
+ * Constructs a new {@class InstrumentedValidator}.
+ *
+ * @param mockedValidator the {@class Validator} to wrap.
+ */
+ public InstrumentedValidator(Validator mockedValidator) {
+ this(mockedValidator,false);
+ }
+
+ /**
+ * Constructs a new {@class InstrumentedValidator}.
+ *
+ * @param mockedValidator the {@class Validator} to wrap.
+ */
+ public InstrumentedValidator(Validator mockedValidator, boolean resetOnGet) {
+ this.mockedValidator = mockedValidator;
+ this.doReset = resetOnGet;
+ }
+
+ /**
+ * Default constructor without wrapping not supported.
+ *
+ */
+ private InstrumentedValidator(){}
+
+ @Override
+ public ValidationResult validate(String subject, String input, ValidationContext context) {
+ validateCallCount++;
+ return mockedValidator.validate(subject, input, context);
+ }
+
+ /**
+ * Returns the count of times validate was called
+ * @return count of validate() calls
+ */
+ public int getValidateCallCount() {
+ int count = validateCallCount;
+ if (doReset) {
+ validateCallCount = 0;
+ }
+ return count;
+ }
+
+ /**
+ * Resets the count of all calls to 0.
+ */
+ public void resetAll() {
+ validateCallCount = 0;
+ }
+}
[nifi] 02/14: NIFI-7175 removed appveyor and travis builds which
have become unreliable. Added Github Actions based CI build for pull
requests and pushes. Removed unit tests which make unreadable or
unaccesible dirs. Includes windows build as well as ubuntu. we can do
localization based builds later as this is all proven stable.
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a commit to branch support/nifi-1.11.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit 046691840420f952c705aee22f58d5a47f65d291
Author: Joe Witt <jo...@apache.org>
AuthorDate: Thu Feb 20 10:55:34 2020 -0500
NIFI-7175 removed appveyor and travis builds which have become unreliable. Added Github Actions based CI build for pull requests and pushes. Removed unit tests which make unreadable or unaccesible dirs. Includes windows build as well as ubuntu. we can do localization based builds later as this is all proven stable.
Signed-off-by: Pierre Villard <pi...@gmail.com>
This closes #4058.
---
.github/workflows/ci-workflow.yml | 111 ++++++++++++++++++
.travis-output-filters | 15 ---
.travis.sh | 28 -----
.travis.yml | 119 -------------------
README.md | 2 +-
appveyor.yml | 38 ------
.../nifi/processors/standard/TestGetFile.java | 127 ++-------------------
pom.xml | 2 +-
8 files changed, 122 insertions(+), 320 deletions(-)
diff --git a/.github/workflows/ci-workflow.yml b/.github/workflows/ci-workflow.yml
new file mode 100644
index 0000000..54993c6
--- /dev/null
+++ b/.github/workflows/ci-workflow.yml
@@ -0,0 +1,111 @@
+name: ci-workflow
+
+on: [push, pull_request]
+
+jobs:
+ ubuntu-build:
+
+ timeout-minutes: 90
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ java: [ '1.8', '11' ]
+ name: Ubuntu Build NiFi - JDK${{ matrix.java }}
+
+ steps:
+ - name: Checkout Code
+ uses: actions/checkout@v2
+ - name: Check NPM Cache
+ uses: actions/cache@v1.1.2
+ with:
+ path: ~/.npm
+ key: linux-${{ matrix.java }}-npm-${{ hashFiles('**/package-lock.json') }}
+ restore-keys: |
+ linux-${{ matrix.java }}-npm-
+ - name: Check Maven Com Cache
+ uses: actions/cache@v1.1.2
+ with:
+ path: ~/.m2/repository/com
+ key: linux-${{ matrix.java }}-maven-com-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ linux-${{ matrix.java }}-maven-com-
+ - name: Check Maven Org Cache
+ uses: actions/cache@v1.1.2
+ with:
+ path: ~/.m2/repository/org
+ key: linux-${{ matrix.java }}-maven-org-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ linux-${{ matrix.java }}-maven-org-
+ - name: Check Maven Net Cache
+ uses: actions/cache@v1.1.2
+ with:
+ path: ~/.m2/repository/net
+ key: linux-${{ matrix.java }}-maven-net-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ linux-${{ matrix.java }}-maven-net-
+ - name: Check Maven IO Cache
+ uses: actions/cache@v1.1.2
+ with:
+ path: ~/.m2/repository/io
+ key: linux-${{ matrix.java }}-maven-io-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ linux-${{ matrix.java }}-maven-io-
+ - name: Set up JDK ${{ matrix.java }}
+ uses: actions/setup-java@v1.3.0
+ with:
+ java-version: ${{ matrix.java }}
+ - name: Build with Maven
+ env:
+ MAVEN_OPTS: -Xmx2g -XX:ReservedCodeCacheSize=1g -XX:+UseG1GC -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN
+ run: |
+ mvn -version
+ mvn -T 1C install -B -Pcontrib-check,include-grpc -Ddir-only -ntp -ff
+ rm -rf ~/.m2/repository/org/apache/nifi
+
+ windows-build:
+
+ runs-on: windows-latest
+ name: Windows Build NiFi - JDK 1.8
+
+ steps:
+ - name: Checkout Code
+ uses: actions/checkout@v2
+ - name: Check Maven Com Cache
+ uses: actions/cache@v1.1.2
+ with:
+ path: ~\.m2\repository\com
+ key: windows-maven-com-${{ hashFiles('**\pom.xml') }}
+ restore-keys: |
+ windows-maven-com-
+ - name: Check Maven Org Cache
+ uses: actions/cache@v1.1.2
+ with:
+ path: ~\.m2\repository\org
+ key: windows-maven-org-${{ hashFiles('**\pom.xml') }}
+ restore-keys: |
+ windows-maven-org-
+ - name: Check Maven Net Cache
+ uses: actions/cache@v1.1.2
+ with:
+ path: ~\.m2\repository\net
+ key: windows-maven-net-${{ hashFiles('**\pom.xml') }}
+ restore-keys: |
+ windows-maven-net-
+ - name: Check Maven IO Cache
+ uses: actions/cache@v1.1.2
+ with:
+ path: ~\.m2\repository\io
+ key: windows-maven-io-${{ hashFiles('**\pom.xml') }}
+ restore-keys: |
+ windows-maven-io-
+ - name: Set up JDK 1.8
+ uses: actions/setup-java@v1.3.0
+ with:
+ java-version: 1.8
+ - name: Build with Maven
+ env:
+ MAVEN_OPTS: -Xmx2g -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN
+ run: |
+ mvn -version
+ mvn -T 1C install -B -Ddir-only -ntp -ff -DskipTests
+ Remove-Item -path '~\.m2\repository\org\apache\nifi' -recurse -force
\ No newline at end of file
diff --git a/.travis-output-filters b/.travis-output-filters
deleted file mode 100644
index f401281..0000000
--- a/.travis-output-filters
+++ /dev/null
@@ -1,15 +0,0 @@
-Downloading from
-Copying
-Progress
-Resolving deltas:
-[WARNING] *****************************************************************
-[WARNING] * Your build is requesting parallel execution, but project *
-[WARNING] * contains the following plugin(s) that have goals not marked *
-[WARNING] * as @threadSafe to support parallel building. *
-[WARNING] * While this /may/ work fine, please look for plugin updates *
-[WARNING] * and/or request plugins be made thread-safe. *
-[WARNING] * If reporting an issue, report it against the plugin in *
-[WARNING] * question, not against maven-core *
-[WARNING] The following plugins are not marked @threadSafe in
-[WARNING] org.apache.rat:apache-rat-plugin
-[WARNING] Enable debug to see more precisely which goals are not marked @threadSafe.
diff --git a/.travis.sh b/.travis.sh
deleted file mode 100755
index adeae67..0000000
--- a/.travis.sh
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-
-# Replace variables seems to be the only option to pass proper values to surefire
-# Note: The reason the sed is done as part of script is to ensure the pom hack
-# won't affect the 'clean install' above
-set -x
-if [ "$USER_LANGUAGE" != "default" ]; then
- sed -i.bak -e "s|<maven\.surefire\.arguments/>|<maven\.surefire\.arguments>-Duser.language=${USER_LANGUAGE} -Duser.region=${USER_REGION}</maven\.surefire\.arguments>|" pom.xml
- diff pom.xml pom.xml.bak
- rm pom.xml.bak
-fi
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index a5fd764..0000000
--- a/.travis.yml
+++ /dev/null
@@ -1,119 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-language: java
-
-matrix:
- include:
- - stage: "Cache"
- name: "Cache Dependencies - no explicit jdk specified"
- script: >-
- if [ ! -d "$HOME"/.m2/repository ] || [ ! "$(ls -A "$HOME"/.m2/repository)" ] ; then
- echo "Cached Maven repository does not exist, downloading dependencies..."
- mvn -V -T 1C -pl !./nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-nar,!./nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors dependency:go-offline -Dsilent -ntp --fail-never \
- | grep -v -F -f .travis-output-filters -e "Failed to execute goal on project"
- else
- echo "Cached Maven repository exists, skipping dependency cache"
- fi
- - stage: "Cache"
- name: "Cache Dependencies - openjdk8 & openjdk11"
- jdk:
- - openjdk8
- - openjdk11
- script: >-
- if [ ! -d "$HOME"/.m2/repository ] || [ ! "$(ls -A "$HOME"/.m2/repository)" ] ; then
- echo "Cached Maven repository does not exist, downloading dependencies..."
- mvn -V -T 1C -pl !./nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-nar,!./nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors dependency:go-offline -Dsilent -ntp --fail-never \
- | grep -v -F -f .travis-output-filters -e "Failed to execute goal on project"
- else
- echo "Cached Maven repository exists, skipping dependency cache"
- fi
-
- # For the build stages, use -pl to exclude the following from the build to reduce build time:
- # non-api nar modules
- # assembly modules
- # nifi-maven-archetype directories that have "__" in the name
- - stage: "Build"
- name: "Build Java 8 EN"
- jdk: openjdk8
- script: >-
- mvn clean install -V -T 1C
- -pl `find . -type d \( -name "*-nar" -or -name "*-assembly" -or -name "*hive3*" -or -name "nifi-system-test*" \) -and -not -name "*api-nar" -and -not -path "*/target/*" -and -not -name "*__*" -printf "!./%P,"`
- -Pcontrib-check,include-grpc -Ddir-only
- -Dmaven.surefire.arguments="-Duser.language=en -Duser.region=US"
- | grep -v -F -f .travis-output-filters
- && test ${PIPESTATUS[0]} -eq 0
- - stage: "Build"
- name: "Build Java 8 FR"
- jdk: openjdk8
- script: >-
- mvn clean install -V -T 1C
- -pl `find . -type d \( -name "*-nar" -or -name "*-assembly" -or -name "*hive3*" -or -name "nifi-system-test*" \) -and -not -name "*api-nar" -and -not -path "*/target/*" -and -not -name "*__*" -printf "!./%P,"`
- -Pcontrib-check,include-grpc -Ddir-only
- -Dmaven.surefire.arguments="-Duser.language=fr -Duser.region=FR"
- | grep -v -F -f .travis-output-filters
- && test ${PIPESTATUS[0]} -eq 0
- - stage: "Build"
- name: "Build Java 8 JA"
- jdk: openjdk8
- script: >-
- mvn clean install -V -T 1C
- -pl `find . -type d \( -name "*-nar" -or -name "*-assembly" -or -name "*hive3*" -or -name "nifi-system-test*" \) -and -not -name "*api-nar" -and -not -path "*/target/*" -and -not -name "*__*" -printf "!./%P,"`
- -Pcontrib-check,include-grpc -Ddir-only
- -Dmaven.surefire.arguments="-Duser.language=ja -Duser.region=JP"
- | grep -v -F -f .travis-output-filters
- && test ${PIPESTATUS[0]} -eq 0
- - stage: "Build"
- name: "Build Java 11 EN"
- jdk: openjdk11
- script: >-
- mvn clean install -V -T 1C
- -pl `find . -type d \( -name "*-nar" -or -name "*-assembly" -or -name "*hive3*" -or -name "nifi-system-test*" \) -and -not -name "*api-nar" -and -not -path "*/target/*" -and -not -name "*__*" -printf "!./%P,"`
- -Pcontrib-check,include-grpc -Ddir-only
- -Dmaven.surefire.arguments="-Duser.language=en -Duser.region=US"
- | grep -v -F -f .travis-output-filters
- && test ${PIPESTATUS[0]} -eq 0
-
-os:
- - linux
-
-# Caches mvn repository in order to speed up builds
-cache:
- directories:
- - $HOME/.m2
- - $HOME/.npm
-
-before_cache:
- # Remove nifi repo again to save travis from caching it
- - rm -rf $HOME/.m2/repository/org/apache/nifi/
-
-# before_install aids in a couple workarounds for issues within the Travis-CI environment
-# 1. Workaround for buffer overflow issues with OpenJDK versions of java as per https://github.com/travis-ci/travis-ci/issues/5227#issuecomment-165135711
-# 2. Workaround for non-existent Maven repository as per: https://github.com/travis-ci/travis-ci/issues/4629#issuecomment-131214465
-before_install:
- - sudo hostname "$(hostname | cut -c1-63)"
- - sed -e "s/^\\(127\\.0\\.0\\.1.*\\)/\\1 $(hostname | cut -c1-63)/" /etc/hosts | sudo tee /etc/hosts
- - sed -i.bak -e 's|https://nexus.codehaus.org/snapshots/|https://oss.sonatype.org/content/repositories/codehaus-snapshots/|g' ~/.m2/settings.xml
- - echo "MAVEN_OPTS='-Xmx2g -XX:+UseG1GC'" > ~/.mavenrc
- - cat ~/.mavenrc
- # Remove nifi repo again to save travis from caching it
- - rm -rf $HOME/.m2/repository/org/apache/nifi/
- # Install Maven 3.6.1
- - wget -O /tmp/apache-maven-3.6.1-bin.tar.gz https://archive.apache.org/dist/maven/maven-3/3.6.1/binaries/apache-maven-3.6.1-bin.tar.gz
- - tar xzf /tmp/apache-maven-3.6.1-bin.tar.gz -C /tmp
- - export M2_HOME=/tmp/apache-maven-3.6.1
- - export PATH=$M2_HOME/bin:$PATH
-
-install: true
diff --git a/README.md b/README.md
index 410efbb..72ca973 100644
--- a/README.md
+++ b/README.md
@@ -14,7 +14,7 @@
-->
[<img src="https://nifi.apache.org/assets/images/apache-nifi-logo.svg" width="300" height="126" alt="Apache NiFi"/>][nifi]
-[](https://travis-ci.org/apache/nifi)
+[
[](https://hub.docker.com/r/apache/nifi/)
[](https://nifi.apache.org/download.html)
[](https://s.apache.org/nifi-community-slack)
diff --git a/appveyor.yml b/appveyor.yml
deleted file mode 100644
index 3e31c19..0000000
--- a/appveyor.yml
+++ /dev/null
@@ -1,38 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: '1.0.0-SNAPSHOT-{build}'
-os: Windows Server 2012
-environment:
- JAVA_HOME: C:\Program Files\Java\jdk1.8.0
-install:
- - ps: |
- Add-Type -AssemblyName System.IO.Compression.FileSystem
- if (!(Test-Path -Path "C:\maven" )) {
- (new-object System.Net.WebClient).DownloadFile(
- 'http://www.us.apache.org/dist/maven/maven-3/3.1.1/binaries/apache-maven-3.1.1-bin.zip',
- 'C:\maven-bin.zip'
- )
- [System.IO.Compression.ZipFile]::ExtractToDirectory("C:\maven-bin.zip", "C:\maven")
- }
- - cmd: SET M2_HOME=C:\maven\apache-maven-3.1.1
- - cmd: SET PATH=%M2_HOME%\bin;%JAVA_HOME%\bin;%PATH%%
- - cmd: SET MAVEN_OPTS=-XX:MaxPermSize=2g -Xmx4g
- - cmd: SET JAVA_OPTS=-XX:MaxPermSize=2g -Xmx4g
-build_script:
- - mvn clean package --batch-mode -DskipTests -Ddir-only
-cache:
- - C:\maven\
- - C:\Users\appveyor\.m2
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGetFile.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGetFile.java
index 7ffdfd8..bfa075a 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGetFile.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGetFile.java
@@ -16,114 +16,30 @@
*/
package org.apache.nifi.processors.standard;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.Test;
import java.io.File;
import java.io.IOException;
-import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
-import java.nio.file.attribute.PosixFilePermission;
-import java.nio.file.attribute.PosixFilePermissions;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
-import java.util.HashSet;
import java.util.List;
import java.util.Locale;
-import java.util.Set;
-import org.apache.nifi.flowfile.attributes.CoreAttributes;
-import org.apache.nifi.util.MockFlowFile;
-import org.apache.nifi.util.TestRunner;
-import org.apache.nifi.util.TestRunners;
-import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
public class TestGetFile {
@Test
- public void testWithInaccessibleDir() throws IOException {
- // Some systems don't support POSIX (Windows) and will fail if run. Should ignore the test in that event
- if (!FileSystems.getDefault().supportedFileAttributeViews().contains("posix")) {
- return;
- }
- File inaccessibleDir = new File("target/inaccessible");
- inaccessibleDir.deleteOnExit();
- inaccessibleDir.mkdir();
- Set<PosixFilePermission> posixFilePermissions = new HashSet<>();
- Files.setPosixFilePermissions(inaccessibleDir.toPath(), posixFilePermissions);
- final TestRunner runner = TestRunners.newTestRunner(new GetFile());
- runner.setProperty(GetFile.DIRECTORY, inaccessibleDir.getAbsolutePath());
- try {
- runner.run();
- fail();
- } catch (AssertionError e) {
- assertTrue(e.getCause().getMessage()
- .endsWith("does not have sufficient permissions (i.e., not writable and readable)"));
- }
- }
-
- @Test
- public void testWithUnreadableDir() throws IOException {
- // Some systems don't support POSIX (Windows) and will fail if run. Should ignore the test in that event
- if (!FileSystems.getDefault().supportedFileAttributeViews().contains("posix")) {
- return;
- }
- File unreadableDir = new File("target/unreadable");
- unreadableDir.deleteOnExit();
- unreadableDir.mkdir();
- Set<PosixFilePermission> posixFilePermissions = new HashSet<>();
- posixFilePermissions.add(PosixFilePermission.GROUP_EXECUTE);
- posixFilePermissions.add(PosixFilePermission.GROUP_WRITE);
- posixFilePermissions.add(PosixFilePermission.OTHERS_EXECUTE);
- posixFilePermissions.add(PosixFilePermission.OTHERS_WRITE);
- posixFilePermissions.add(PosixFilePermission.OWNER_EXECUTE);
- posixFilePermissions.add(PosixFilePermission.OWNER_WRITE);
- Files.setPosixFilePermissions(unreadableDir.toPath(), posixFilePermissions);
- final TestRunner runner = TestRunners.newTestRunner(new GetFile());
- runner.setProperty(GetFile.DIRECTORY, unreadableDir.getAbsolutePath());
- try {
- runner.run();
- fail();
- } catch (AssertionError e) {
- Throwable ex = e.getCause();
- assertTrue(e.getCause().getMessage()
- .endsWith("does not have sufficient permissions (i.e., not writable and readable)"));
- }
- }
-
- @Test
- public void testWithUnwritableDir() throws IOException {
- // Some systems don't support POSIX (Windows) and will fail if run. Should ignore the test in that event
- if (!FileSystems.getDefault().supportedFileAttributeViews().contains("posix")) {
- return;
- }
- File unwritableDir = new File("target/unwritable");
- unwritableDir.deleteOnExit();
- unwritableDir.mkdir();
- Set<PosixFilePermission> posixFilePermissions = new HashSet<>();
- posixFilePermissions.add(PosixFilePermission.GROUP_EXECUTE);
- posixFilePermissions.add(PosixFilePermission.GROUP_READ);
- posixFilePermissions.add(PosixFilePermission.OTHERS_EXECUTE);
- posixFilePermissions.add(PosixFilePermission.OTHERS_READ);
- posixFilePermissions.add(PosixFilePermission.OWNER_EXECUTE);
- posixFilePermissions.add(PosixFilePermission.OWNER_READ);
- Files.setPosixFilePermissions(unwritableDir.toPath(), posixFilePermissions);
- final TestRunner runner = TestRunners.newTestRunner(new GetFile());
- runner.setProperty(GetFile.DIRECTORY, unwritableDir.getAbsolutePath());
- try {
- runner.run();
- fail();
- } catch (AssertionError e) {
- assertTrue(e.getCause().getMessage()
- .endsWith("does not have sufficient permissions (i.e., not writable and readable)"));
- }
- }
-
- @Test
public void testFilePickedUp() throws IOException {
final File directory = new File("target/test/data/in");
deleteDirectory(directory);
@@ -237,29 +153,6 @@ public class TestGetFile {
} catch (Exception doNothing) {
}
- boolean verifyPermissions = false;
- try {
- /* If you mount an NTFS partition in Linux, you are unable to change the permissions of the files,
- * because every file has the same permissions, controlled by the 'fmask' and 'dmask' mount options.
- * Executing a chmod command will not fail, but it does not change the file's permissions.
- * From Java perspective the NTFS mount point, as a FileStore supports the 'unix' and 'posix' file
- * attribute views, but the setPosixFilePermissions() has no effect.
- *
- * If you set verifyPermissions to true without the following extra check, the test case will fail
- * on a file system, where Nifi source is located on a NTFS mount point in Linux.
- * The purpose of the extra check is to ensure, that setPosixFilePermissions() changes the file's
- * permissions, and set verifyPermissions, after we are convinced.
- */
- Set<PosixFilePermission> perms = PosixFilePermissions.fromString("r--r-----");
- Files.setPosixFilePermissions(targetPath, perms);
- Set<PosixFilePermission> permsAfterSet = Files.getPosixFilePermissions(targetPath);
- if (perms.equals(permsAfterSet)) {
- verifyPermissions = true;
- }
-
- } catch (Exception doNothing) {
- }
-
final TestRunner runner = TestRunners.newTestRunner(new GetFile());
runner.setProperty(GetFile.DIRECTORY, "target/test/data/in");
runner.run();
@@ -276,8 +169,6 @@ public class TestGetFile {
fail();
}
}
- if (verifyPermissions) {
- successFiles.get(0).assertAttributeEquals("file.permissions", "r--r-----");
- }
+ //permissions are not verified as these are very environmentally specific
}
}
diff --git a/pom.xml b/pom.xml
index 1fe9527..a19223c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -691,7 +691,7 @@
<exclude>nbactions.xml</exclude> <!-- courtesy excludes for netbeans users -->
<exclude>DEPENDENCIES</exclude> <!-- auto generated file by apache's maven config while building sources.zip -->
<exclude>.github/PULL_REQUEST_TEMPLATE.md</exclude> <!-- PR Template for GitHub that does not have a mechanism of including comments -->
- <exclude>.travis-output-filters</exclude> <!-- grep patterns to exclude from travis output -->
+ <exclude>.github/workflows/ci-workflow.yml</exclude>
</excludes>
</configuration>
</plugin>
[nifi] 13/14: NIFI-7165 Fixed TLS Toolkit Guide flags with default
validity days.
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a commit to branch support/nifi-1.11.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit ea138984efb25fdd756c724a4a2adac65cd49462
Author: Andy LoPresto <al...@apache.org>
AuthorDate: Wed Feb 19 17:21:23 2020 -0800
NIFI-7165 Fixed TLS Toolkit Guide flags with default validity days.
Signed-off-by: Pierre Villard <pi...@gmail.com>
This closes #4064.
---
nifi-docs/src/main/asciidoc/toolkit-guide.adoc | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/nifi-docs/src/main/asciidoc/toolkit-guide.adoc b/nifi-docs/src/main/asciidoc/toolkit-guide.adoc
index baaf555..05459f0 100644
--- a/nifi-docs/src/main/asciidoc/toolkit-guide.adoc
+++ b/nifi-docs/src/main/asciidoc/toolkit-guide.adoc
@@ -758,7 +758,7 @@ The following are available options:
* `-B`,`--clientCertPassword <arg>` Password for client certificate. Must either be one value or one for each client DN (auto-generate if not specified)
* `-c`,`--certificateAuthorityHostname <arg>` Hostname of NiFi Certificate Authority (default: `localhost`)
* `-C`,`--clientCertDn <arg>` Generate client certificate suitable for use in browser with specified DN (Can be specified multiple times)
-* `-d`,`--days <arg>` Number of days issued certificate should be valid for (default: `1095`)
+* `-d`,`--days <arg>` Number of days issued certificate should be valid for (default: `825`)
* `-f`,`--nifiPropertiesFile <arg>` Base _nifi.properties_ file to update (Embedded file identical to the one in a default NiFi install will be used if not specified)
* `-g`,`--differentKeyAndKeystorePasswords` Use different generated password for the key and the keystore
* `-G`,`--globalPortSequence <arg>` Use sequential ports that are calculated for all hosts according to the provided hostname expressions (Can be specified multiple times, MUST BE SAME FROM RUN TO RUN)
@@ -819,7 +819,7 @@ The following are available options:
* `-a`,`--keyAlgorithm <arg>` Algorithm to use for generated keys (default: `RSA`)
* `--configJsonIn <arg>` The place to read configuration info from (defaults to the value of configJson), implies useConfigJson if set (default: `configJson` value)
-* `-d`,`--days <arg>` Number of days issued certificate should be valid for (default: `1095`)
+* `-d`,`--days <arg>` Number of days issued certificate should be valid for (default: `825`)
* `-D`,`--dn <arg>` The dn to use for the CA certificate (default: `CN=YOUR_CA_HOSTNAME,OU=NIFI`)
* `-f`,`--configJson <arg>` The place to write configuration info (default: `config.json`)
* `-F`,`--useConfigJson` Flag specifying that all configuration is read from `configJson` to facilitate automated use (otherwise `configJson` will only be written to)
[nifi] 06/14: NIFI-7136 Added autocomplete="off" to login password
input (#4055)
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a commit to branch support/nifi-1.11.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit c276ac5f27674a45243632d22b11b76f089b15ca
Author: M Tien <56...@users.noreply.github.com>
AuthorDate: Fri Feb 14 20:46:08 2020 -0800
NIFI-7136 Added autocomplete="off" to login password input (#4055)
NIFI-7136 Added autocomplete="off" to login password input
Updated nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/login/login-form.jsp
Co-authored-by: Pierre Villard <pi...@gmail.com>
Signed-off-by: Andy LoPresto <al...@apache.org>
---
.../nifi-web-ui/src/main/webapp/WEB-INF/partials/login/login-form.jsp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/login/login-form.jsp b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/login/login-form.jsp
index 9f9a660..6339664 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/login/login-form.jsp
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/login/login-form.jsp
@@ -26,7 +26,7 @@
<div class="setting">
<div class="setting-name">Password</div>
<div class="setting-field">
- <input type="password" placeholder="password" id="password"/>
+ <input type="password" placeholder="password" id="password" autocomplete="off"/>
</div>
</div>
-</div>
\ No newline at end of file
+</div>
[nifi] 09/14: NIFI-7142: Automatically handle schema drift in the
PutKudu processor
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a commit to branch support/nifi-1.11.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit 1a07d2634cd4ef1ce7a4a71069170415221b8c40
Author: Grant Henke <gh...@cloudera.com>
AuthorDate: Thu Feb 13 09:24:08 2020 -0600
NIFI-7142: Automatically handle schema drift in the PutKudu processor
Adds a boolean property to the PutKudu processor to optionally
enable automatic schema drift handling.
If set to true, when fields with names that are not in the target
Kudu table are encountered, the Kudu table will be altered to
include new columns for those fields.
Signed-off-by: Pierre Villard <pi...@gmail.com>
This closes #4053.
---
.../processors/kudu/AbstractKuduProcessor.java | 30 +++++++++
.../org/apache/nifi/processors/kudu/PutKudu.java | 74 ++++++++++++++++++++--
.../org/apache/nifi/processors/kudu/ITPutKudu.java | 29 +++++++--
3 files changed, 121 insertions(+), 12 deletions(-)
diff --git a/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/main/java/org/apache/nifi/processors/kudu/AbstractKuduProcessor.java b/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/main/java/org/apache/nifi/processors/kudu/AbstractKuduProcessor.java
index 166d233..570b86d 100644
--- a/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/main/java/org/apache/nifi/processors/kudu/AbstractKuduProcessor.java
+++ b/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/main/java/org/apache/nifi/processors/kudu/AbstractKuduProcessor.java
@@ -45,6 +45,7 @@ import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.security.krb.KerberosAction;
import org.apache.nifi.security.krb.KerberosKeytabUser;
import org.apache.nifi.security.krb.KerberosUser;
+import org.apache.nifi.serialization.record.DataType;
import org.apache.nifi.serialization.record.Record;
import javax.security.auth.login.LoginException;
@@ -231,6 +232,35 @@ public abstract class AbstractKuduProcessor extends AbstractProcessor {
}
}
+ /**
+ * Converts a NiFi DataType to it's equivalent Kudu Type.
+ */
+ protected Type toKuduType(DataType nifiType) {
+ switch (nifiType.getFieldType()) {
+ case BOOLEAN:
+ return Type.BOOL;
+ case BYTE:
+ return Type.INT8;
+ case SHORT:
+ return Type.INT16;
+ case INT:
+ return Type.INT32;
+ case LONG:
+ return Type.INT64;
+ case FLOAT:
+ return Type.FLOAT;
+ case DOUBLE:
+ return Type.DOUBLE;
+ case TIMESTAMP:
+ return Type.UNIXTIME_MICROS;
+ case CHAR:
+ case STRING:
+ return Type.STRING;
+ default:
+ throw new IllegalArgumentException(String.format("unsupported type %s", nifiType));
+ }
+ }
+
private int getColumnIndex(Schema columns, String colName) {
try {
return columns.getColumnIndex(colName);
diff --git a/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/main/java/org/apache/nifi/processors/kudu/PutKudu.java b/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/main/java/org/apache/nifi/processors/kudu/PutKudu.java
index 0bb5882..aedac37 100644
--- a/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/main/java/org/apache/nifi/processors/kudu/PutKudu.java
+++ b/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/main/java/org/apache/nifi/processors/kudu/PutKudu.java
@@ -17,7 +17,10 @@
package org.apache.nifi.processors.kudu;
+import org.apache.kudu.Schema;
+import org.apache.kudu.client.AlterTableOptions;
import org.apache.kudu.client.KuduClient;
+import org.apache.kudu.client.KuduException;
import org.apache.kudu.client.KuduTable;
import org.apache.kudu.client.KuduSession;
import org.apache.kudu.client.Operation;
@@ -34,6 +37,7 @@ import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.annotation.lifecycle.OnScheduled;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.PropertyDescriptor.Builder;
+import org.apache.nifi.components.PropertyValue;
import org.apache.nifi.expression.ExpressionLanguageScope;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.processor.ProcessContext;
@@ -46,6 +50,7 @@ import org.apache.nifi.security.krb.KerberosUser;
import org.apache.nifi.serialization.RecordReader;
import org.apache.nifi.serialization.RecordReaderFactory;
import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordField;
import org.apache.nifi.serialization.record.RecordSet;
import javax.security.auth.login.LoginException;
@@ -59,6 +64,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
@EventDriven
@SupportsBatching
@@ -108,6 +114,16 @@ public class PutKudu extends AbstractKuduProcessor {
.addValidator(StandardValidators.BOOLEAN_VALIDATOR)
.build();
+ protected static final PropertyDescriptor HANDLE_SCHEMA_DRIFT = new Builder()
+ .name("Handle Schema Drift")
+ .description("If set to true, when fields with names that are not in the target Kudu table " +
+ "are encountered, the Kudu table will be altered to include new columns for those fields.")
+ .defaultValue("false")
+ .required(true)
+ .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
+ .addValidator(StandardValidators.BOOLEAN_VALIDATOR)
+ .build();
+
protected static final PropertyDescriptor INSERT_OPERATION = new Builder()
.name("Insert Operation")
.displayName("Kudu Operation Type")
@@ -158,7 +174,6 @@ public class PutKudu extends AbstractKuduProcessor {
.name("Ignore NULL")
.description("Ignore NULL on Kudu Put Operation, Update only non-Null columns if set true")
.defaultValue("false")
- .allowableValues("true", "false")
.required(true)
.addValidator(StandardValidators.BOOLEAN_VALIDATOR)
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
@@ -188,6 +203,7 @@ public class PutKudu extends AbstractKuduProcessor {
properties.add(KERBEROS_CREDENTIALS_SERVICE);
properties.add(SKIP_HEAD_LINE);
properties.add(LOWERCASE_FIELD_NAMES);
+ properties.add(HANDLE_SCHEMA_DRIFT);
properties.add(RECORD_READER);
properties.add(INSERT_OPERATION);
properties.add(FLUSH_MODE);
@@ -251,16 +267,52 @@ public class PutKudu extends AbstractKuduProcessor {
OperationType prevOperationType = OperationType.INSERT;
final List<RowError> pendingRowErrors = new ArrayList<>();
for (FlowFile flowFile : flowFiles) {
- final String tableName = context.getProperty(TABLE_NAME).evaluateAttributeExpressions(flowFile).getValue();
- final OperationType operationType = OperationType.valueOf(context.getProperty(INSERT_OPERATION).evaluateAttributeExpressions(flowFile).getValue());
- final Boolean ignoreNull = Boolean.valueOf(context.getProperty(IGNORE_NULL).evaluateAttributeExpressions(flowFile).getValue());
- final Boolean lowercaseFields = Boolean.valueOf(context.getProperty(LOWERCASE_FIELD_NAMES).evaluateAttributeExpressions(flowFile).getValue());
-
try (final InputStream in = session.read(flowFile);
final RecordReader recordReader = recordReaderFactory.createRecordReader(flowFile, in, getLogger())) {
+
+ final String tableName = getEvaluatedProperty(TABLE_NAME, context, flowFile);
+ final OperationType operationType = OperationType.valueOf(getEvaluatedProperty(INSERT_OPERATION, context, flowFile));
+ final Boolean ignoreNull = Boolean.valueOf(getEvaluatedProperty(IGNORE_NULL, context, flowFile));
+ final Boolean lowercaseFields = Boolean.valueOf(getEvaluatedProperty(LOWERCASE_FIELD_NAMES, context, flowFile));
+ final Boolean handleSchemaDrift = Boolean.valueOf(getEvaluatedProperty(HANDLE_SCHEMA_DRIFT, context, flowFile));
+
final RecordSet recordSet = recordReader.createRecordSet();
final List<String> fieldNames = recordReader.getSchema().getFieldNames();
- final KuduTable kuduTable = kuduClient.openTable(tableName);
+ KuduTable kuduTable = kuduClient.openTable(tableName);
+
+ // If handleSchemaDrift is true, check for any missing columns and alter the Kudu table to add them.
+ if (handleSchemaDrift) {
+ final Schema schema = kuduTable.getSchema();
+ Stream<RecordField> fields = recordReader.getSchema().getFields().stream();
+ List<RecordField> missing = fields.filter(field -> !schema.hasColumn(
+ lowercaseFields ? field.getFieldName().toLowerCase() : field.getFieldName()))
+ .collect(Collectors.toList());
+ if (!missing.isEmpty()) {
+ getLogger().info("adding {} columns to table '{}' to handle schema drift",
+ new Object[]{missing.size(), tableName});
+ // Add each column one at a time to avoid failing if some of the missing columns
+ // we created by a concurrent thread or application attempting to handle schema drift.
+ for (RecordField field : missing) {
+ try {
+ String columnName = lowercaseFields ? field.getFieldName().toLowerCase() : field.getFieldName();
+ AlterTableOptions alter = new AlterTableOptions();
+ alter.addNullableColumn(columnName, toKuduType(field.getDataType()));
+ kuduClient.alterTable(tableName, alter);
+ } catch (KuduException e) {
+ // Ignore the exception if the column already exists due to concurrent
+ // threads or applications attempting to handle schema drift.
+ if (e.getStatus().isAlreadyPresent()) {
+ getLogger().info("column already exists in table '{}' while handling schema drift",
+ new Object[]{tableName});
+ } else {
+ throw new ProcessException(e);
+ }
+ }
+ }
+ // Re-open the table to get the new schema.
+ kuduTable = kuduClient.openTable(tableName);
+ }
+ }
// In the case of INSERT_IGNORE the Kudu session is modified to ignore row errors.
// Because the session is shared across flow files, for batching efficiency, we
@@ -348,6 +400,14 @@ public class PutKudu extends AbstractKuduProcessor {
session.adjustCounter("Records Inserted", totalCount, false);
}
+ private String getEvaluatedProperty(PropertyDescriptor property, ProcessContext context, FlowFile flowFile) {
+ PropertyValue evaluatedProperty = context.getProperty(property).evaluateAttributeExpressions(flowFile);
+ if (property.isRequired() && evaluatedProperty == null) {
+ throw new ProcessException(String.format("Property `%s` is required but evaluated to null", property.getDisplayName()));
+ }
+ return evaluatedProperty.getValue();
+ }
+
protected KuduSession createKuduSession(final KuduClient client) {
final KuduSession kuduSession = client.newSession();
kuduSession.setMutationBufferSpace(batchSize);
diff --git a/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/test/java/org/apache/nifi/processors/kudu/ITPutKudu.java b/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/test/java/org/apache/nifi/processors/kudu/ITPutKudu.java
index 1de5c23..eee5e60 100644
--- a/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/test/java/org/apache/nifi/processors/kudu/ITPutKudu.java
+++ b/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/test/java/org/apache/nifi/processors/kudu/ITPutKudu.java
@@ -99,9 +99,8 @@ public class ITPutKudu {
KuduClient client = harness.getClient();
List<ColumnSchema> columns = new ArrayList<>();
columns.add(new ColumnSchema.ColumnSchemaBuilder("id", Type.INT32).key(true).build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder("stringVal", Type.STRING).build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder("num32Val", Type.INT32).build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder("doubleVal", Type.DOUBLE).build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("stringval", Type.STRING).build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("num32val", Type.INT32).build());
Schema schema = new Schema(columns);
CreateTableOptions opts = new CreateTableOptions()
.addHashPartitions(Collections.singletonList("id"), 4);
@@ -113,10 +112,12 @@ public class ITPutKudu {
readerFactory.addSchemaField("id", RecordFieldType.INT);
readerFactory.addSchemaField("stringVal", RecordFieldType.STRING);
readerFactory.addSchemaField("num32Val", RecordFieldType.INT);
+ // Add two extra columns to test handleSchemaDrift = true.
readerFactory.addSchemaField("doubleVal", RecordFieldType.DOUBLE);
+ readerFactory.addSchemaField("floatVal", RecordFieldType.FLOAT);
for (int i = 0; i < numOfRecord; i++) {
- readerFactory.addRecord(i, "val_" + i, 1000 + i, 100.88 + i);
+ readerFactory.addRecord(i, "val_" + i, 1000 + i, 100.88 + i, 100.88 + i);
}
testRunner.addControllerService("mock-reader-factory", readerFactory);
@@ -139,6 +140,18 @@ public class ITPutKudu {
testRunner.setProperty(PutKudu.BATCH_SIZE, "10");
testRunner.setProperty(PutKudu.FLOWFILE_BATCH_SIZE, "2");
+ // Don't ignore null values.
+ flowFileAttributes.put("kudu.ignore.null", "false");
+ testRunner.setProperty(PutKudu.IGNORE_NULL, "${kudu.ignore.null}");
+
+ // Enable lowercase handling.
+ flowFileAttributes.put("kudu.lowercase.field.names", "true");
+ testRunner.setProperty(PutKudu.LOWERCASE_FIELD_NAMES, "${kudu.lowercase.field.names}");
+
+ // Enable schema drift handling.
+ flowFileAttributes.put("kudu.handle.schema.drift", "true");
+ testRunner.setProperty(PutKudu.HANDLE_SCHEMA_DRIFT, "${kudu.handle.schema.drift}");
+
// Increase the thread count to better simulate a production environment
testRunner.setThreadCount(4);
@@ -163,9 +176,15 @@ public class ITPutKudu {
final ProvenanceEventRecord provEvent = provEvents.get(0);
Assert.assertEquals(ProvenanceEventType.SEND, provEvent.getEventType());
- // Verify Kudu record count.
KuduClient client = harness.getClient();
KuduTable kuduTable = client.openTable(DEFAULT_TABLE_NAME);
+
+ // Verify the extra field was added.
+ Assert.assertEquals(5, kuduTable.getSchema().getColumnCount());
+ Assert.assertTrue(kuduTable.getSchema().hasColumn("doubleval"));
+ Assert.assertTrue(kuduTable.getSchema().hasColumn("floatval"));
+
+ // Verify Kudu record count.
KuduScanner scanner = client.newScannerBuilder(kuduTable).build();
int count = 0;
for (RowResult unused : scanner) {
[nifi] 11/14: NIFI-6551: Improve PutKudu timestamp handling
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a commit to branch support/nifi-1.11.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit 6b117367fd94f18e2467bb536f233701509706fb
Author: Grant Henke <gh...@cloudera.com>
AuthorDate: Thu Feb 13 17:03:20 2020 -0600
NIFI-6551: Improve PutKudu timestamp handling
Uses `DataTypeUtils.toTimestamp` when writing to Kudu
timestamp (`UNIXTIME_MICROS`) columns. This allows
us to use the `row.addTimestamp` API and get much more
intuitive and predictable timestamp ingest behavior.
Signed-off-by: Pierre Villard <pi...@gmail.com>
This closes #4054.
---
.../processors/kudu/AbstractKuduProcessor.java | 51 ++++++++++++----------
.../org/apache/nifi/processors/kudu/ITPutKudu.java | 12 +++--
.../apache/nifi/processors/kudu/TestPutKudu.java | 5 ++-
3 files changed, 40 insertions(+), 28 deletions(-)
diff --git a/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/main/java/org/apache/nifi/processors/kudu/AbstractKuduProcessor.java b/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/main/java/org/apache/nifi/processors/kudu/AbstractKuduProcessor.java
index 570b86d..9636cf6 100644
--- a/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/main/java/org/apache/nifi/processors/kudu/AbstractKuduProcessor.java
+++ b/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/main/java/org/apache/nifi/processors/kudu/AbstractKuduProcessor.java
@@ -47,9 +47,11 @@ import org.apache.nifi.security.krb.KerberosKeytabUser;
import org.apache.nifi.security.krb.KerberosUser;
import org.apache.nifi.serialization.record.DataType;
import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.util.DataTypeUtils;
import javax.security.auth.login.LoginException;
import java.math.BigDecimal;
+import java.sql.Timestamp;
import java.util.Arrays;
import java.util.concurrent.TimeUnit;
import java.util.List;
@@ -182,47 +184,50 @@ public abstract class AbstractKuduProcessor extends AbstractProcessor {
if (schema.getColumnByIndex(colIdx).isKey()) {
throw new IllegalArgumentException(String.format("Can't set primary key column %s to null ", colName));
} else if(!schema.getColumnByIndex(colIdx).isNullable()) {
- throw new IllegalArgumentException(String.format("Can't set primary key column %s to null ", colName));
+ throw new IllegalArgumentException(String.format("Can't set column %s to null ", colName));
}
if (!ignoreNull) {
row.setNull(colName);
- continue;
}
} else {
- switch (colType.getDataType(colSchema.getTypeAttributes())) {
+ Object value = record.getValue(recordFieldName);
+ switch (colType) {
case BOOL:
- row.addBoolean(colIdx, record.getAsBoolean(recordFieldName));
- break;
- case FLOAT:
- row.addFloat(colIdx, record.getAsFloat(recordFieldName));
- break;
- case DOUBLE:
- row.addDouble(colIdx, record.getAsDouble(recordFieldName));
- break;
- case BINARY:
- row.addBinary(colIdx, record.getAsString(recordFieldName).getBytes());
+ row.addBoolean(colIdx, DataTypeUtils.toBoolean(value, recordFieldName));
break;
case INT8:
- row.addByte(colIdx, record.getAsInt(recordFieldName).byteValue());
+ row.addByte(colIdx, DataTypeUtils.toByte(value, recordFieldName));
break;
case INT16:
- row.addShort(colIdx, record.getAsInt(recordFieldName).shortValue());
+ row.addShort(colIdx, DataTypeUtils.toShort(value, recordFieldName));
break;
case INT32:
- row.addInt(colIdx, record.getAsInt(recordFieldName));
+ row.addInt(colIdx, DataTypeUtils.toInteger(value, recordFieldName));
break;
case INT64:
+ row.addLong(colIdx, DataTypeUtils.toLong(value, recordFieldName));
+ break;
case UNIXTIME_MICROS:
- row.addLong(colIdx, record.getAsLong(recordFieldName));
+ DataType fieldType = record.getSchema().getDataType(recordFieldName).get();
+ Timestamp timestamp = DataTypeUtils.toTimestamp(record.getValue(recordFieldName),
+ () -> DataTypeUtils.getDateFormat(fieldType.getFormat()), recordFieldName);
+ row.addTimestamp(colIdx, timestamp);
break;
case STRING:
- row.addString(colIdx, record.getAsString(recordFieldName));
+ row.addString(colIdx, DataTypeUtils.toString(value, recordFieldName));
+ break;
+ case BINARY:
+ row.addBinary(colIdx, DataTypeUtils.toString(value, recordFieldName).getBytes());
+ break;
+ case FLOAT:
+ row.addFloat(colIdx, DataTypeUtils.toFloat(value, recordFieldName));
+ break;
+ case DOUBLE:
+ row.addDouble(colIdx, DataTypeUtils.toDouble(value, recordFieldName));
break;
- case DECIMAL32:
- case DECIMAL64:
- case DECIMAL128:
- row.addDecimal(colIdx, new BigDecimal(record.getAsString(recordFieldName)));
+ case DECIMAL:
+ row.addDecimal(colIdx, new BigDecimal(DataTypeUtils.toString(value, recordFieldName)));
break;
default:
throw new IllegalStateException(String.format("unknown column type %s", colType));
@@ -293,4 +298,4 @@ public abstract class AbstractKuduProcessor extends AbstractProcessor {
return update;
}
-}
\ No newline at end of file
+}
diff --git a/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/test/java/org/apache/nifi/processors/kudu/ITPutKudu.java b/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/test/java/org/apache/nifi/processors/kudu/ITPutKudu.java
index b3ad170..1be21cb 100644
--- a/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/test/java/org/apache/nifi/processors/kudu/ITPutKudu.java
+++ b/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/test/java/org/apache/nifi/processors/kudu/ITPutKudu.java
@@ -43,6 +43,7 @@ import org.junit.Rule;
import org.junit.Test;
import java.io.IOException;
+import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -54,6 +55,8 @@ public class ITPutKudu {
public static final String DEFAULT_TABLE_NAME = "Nifi-Kudu-Table";
+ public static final Timestamp NOW = new Timestamp(System.currentTimeMillis());
+
// The KuduTestHarness automatically starts and stops a real Kudu cluster
// when each test is run. Kudu persists its on-disk state in a temporary
// directory under a location defined by the environment variable TEST_TMPDIR
@@ -101,6 +104,7 @@ public class ITPutKudu {
columns.add(new ColumnSchema.ColumnSchemaBuilder("id", Type.INT32).key(true).build());
columns.add(new ColumnSchema.ColumnSchemaBuilder("stringval", Type.STRING).build());
columns.add(new ColumnSchema.ColumnSchemaBuilder("num32val", Type.INT32).build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("timestampval", Type.UNIXTIME_MICROS).build());
Schema schema = new Schema(columns);
CreateTableOptions opts = new CreateTableOptions()
.addHashPartitions(Collections.singletonList("id"), 4);
@@ -112,12 +116,13 @@ public class ITPutKudu {
readerFactory.addSchemaField("id", RecordFieldType.INT);
readerFactory.addSchemaField("stringVal", RecordFieldType.STRING);
readerFactory.addSchemaField("num32Val", RecordFieldType.INT);
+ readerFactory.addSchemaField("timestampVal", RecordFieldType.TIMESTAMP);
// Add two extra columns to test handleSchemaDrift = true.
readerFactory.addSchemaField("doubleVal", RecordFieldType.DOUBLE);
readerFactory.addSchemaField("floatVal", RecordFieldType.FLOAT);
for (int i = 0; i < numOfRecord; i++) {
- readerFactory.addRecord(i, "val_" + i, 1000 + i, 100.88 + i, 100.88 + i);
+ readerFactory.addRecord(i, "val_" + i, 1000 + i, NOW, 100.88 + i, 100.88 + i);
}
testRunner.addControllerService("mock-reader-factory", readerFactory);
@@ -183,14 +188,15 @@ public class ITPutKudu {
KuduTable kuduTable = client.openTable(DEFAULT_TABLE_NAME);
// Verify the extra field was added.
- Assert.assertEquals(5, kuduTable.getSchema().getColumnCount());
+ Assert.assertEquals(6, kuduTable.getSchema().getColumnCount());
Assert.assertTrue(kuduTable.getSchema().hasColumn("doubleval"));
Assert.assertTrue(kuduTable.getSchema().hasColumn("floatval"));
// Verify Kudu record count.
KuduScanner scanner = client.newScannerBuilder(kuduTable).build();
int count = 0;
- for (RowResult unused : scanner) {
+ for (RowResult row : scanner) {
+ Assert.assertEquals(NOW, row.getTimestamp("timestampval"));
count++;
}
Assert.assertEquals(recordCount, count);
diff --git a/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/test/java/org/apache/nifi/processors/kudu/TestPutKudu.java b/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/test/java/org/apache/nifi/processors/kudu/TestPutKudu.java
index 9be6952..b55695f 100644
--- a/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/test/java/org/apache/nifi/processors/kudu/TestPutKudu.java
+++ b/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/test/java/org/apache/nifi/processors/kudu/TestPutKudu.java
@@ -60,6 +60,7 @@ import org.mockito.stubbing.OngoingStubbing;
import java.io.IOException;
import java.io.InputStream;
+import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -424,7 +425,7 @@ public class TestPutKudu {
new RecordField(recordIdName, RecordFieldType.BIGINT.getDataType()),
new RecordField("name", RecordFieldType.STRING.getDataType()),
new RecordField("age", RecordFieldType.SHORT.getDataType()),
- new RecordField("updated_at", RecordFieldType.BIGINT.getDataType()),
+ new RecordField("updated_at", RecordFieldType.TIMESTAMP.getDataType()),
new RecordField("score", RecordFieldType.LONG.getDataType())));
Map<String, Object> values = new HashMap<>();
@@ -432,7 +433,7 @@ public class TestPutKudu {
values.put(recordIdName, id);
values.put("name", name);
values.put("age", age);
- values.put("updated_at", System.currentTimeMillis() * 1000);
+ values.put("updated_at", new Timestamp(System.currentTimeMillis()));
values.put("score", 10000L);
processor.buildPartialRow(
kuduSchema,
[nifi] 14/14: NIFI-7175 Fixed core attributes formatting in
Developer Guide. (#4066)
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a commit to branch support/nifi-1.11.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit 97486956e04e7eb940b9cddbd9301913411f1192
Author: Andy LoPresto <al...@users.noreply.github.com>
AuthorDate: Fri Feb 21 10:41:41 2020 -0800
NIFI-7175 Fixed core attributes formatting in Developer Guide. (#4066)
* NIFI-7175 Fixed core attributes formatting in Developer Guide.
* NIFI-7175 Made core attribute names more consistent.
---
nifi-docs/src/main/asciidoc/developer-guide.adoc | 30 ++++++++++++++----------
1 file changed, 17 insertions(+), 13 deletions(-)
diff --git a/nifi-docs/src/main/asciidoc/developer-guide.adoc b/nifi-docs/src/main/asciidoc/developer-guide.adoc
index 2a2c93c..aca8b69 100644
--- a/nifi-docs/src/main/asciidoc/developer-guide.adoc
+++ b/nifi-docs/src/main/asciidoc/developer-guide.adoc
@@ -127,30 +127,34 @@ can change, the FlowFile object is
immutable. Modifications to a FlowFile are made possible by the ProcessSession.
The core attributes for FlowFiles are defined in the `org.apache.nifi.flowfile.attributes.CoreAttributes` enum.
-The most common attributes you'll see are filename, path and uuid. The string in quotes is the value of the
-attribute within the `CoreAttributes` enum.
+The most common attributes you'll see are `filename`, `path` and `uuid`. The string in parentheses is the value of the
+attribute within the `CoreAttributes` enum and how it appears in the UI/API.
-- Filename ("filename"): The filename of the FlowFile. The filename should not contain any directory structure.
+- Filename (`filename`): The filename of the FlowFile. The filename should not contain any directory structure.
-- File Size ("fileSize"): The file size of the FlowFile in bytes.
+- UUID (`uuid`): A Universally Unique Identifier assigned to this FlowFile that distinguishes the FlowFile from other FlowFiles in the system.
-- UUID ("uuid"): A Universally Unique Identifier assigned to this FlowFile that distinguishes the FlowFile from other FlowFiles in the system.
+- Path (`path`): The FlowFile's path indicates the relative directory to which a FlowFile belongs and does not contain the filename.
-- Path ("path"): The FlowFile's path indicates the relative directory to which a FlowFile belongs and does not contain the filename.
+- Absolute Path (`absolute.path`): The FlowFile's absolute path indicates the absolute directory to which a FlowFile belongs and does not contain the filename.
-- Absolute Path ("absolute.path"): The FlowFile's absolute path indicates the absolute directory to which a FlowFile belongs and does not contain the filename.
+- Priority (`priority`): A numeric value indicating the FlowFile priority.
-- Entry Date ("entryDate"): The date and time at which the FlowFile entered the system (i.e., was created). The value of this attribute is a number that represents the number of milliseconds since midnight, Jan. 1, 1970 (UTC).
+- MIME Type (`mime.type`): The MIME Type of this FlowFile.
-- Lineage Start Date ("lineageStartDate"): Any time that a FlowFile is cloned, merged, or split, this results in a "child" FlowFile being created. As those children are then cloned, merged, or split, a chain of ancestors is built. This value represents the date and time at which the oldest ancestor entered the system. Another way to think about this is that this attribute represents the latency of the FlowFile through the system. The value is a number that represents the number of millis [...]
+- Discard Reason (`discard.reason`): Specifies the reason that a FlowFile is being discarded.
-- Priority ("priority"): A numeric value indicating the FlowFile priority.
+- Alternate Identifier (`alternate.identifier`): Indicates an identifier other than the FlowFile's UUID that is known to refer to this FlowFile.
-- MIME Type ("mime.type"): The MIME Type of this FlowFile.
+===== Additional Common Attributes
-- Discard Reason ("discard.reason"): Specifies the reason that a FlowFile is being discarded.
+While these attributes are not members of the `CoreAttributes` enum, they are de facto standards across the system and found on most FlowFiles.
-- Alternative Identifier ("alternate.identifier"): Indicates an identifier other than the FlowFile's UUID that is known to refer to this FlowFile.
+- File Size (`fileSize`): The size of the FlowFile content in bytes.
+
+- Entry Date (`entryDate`): The date and time at which the FlowFile entered the system (i.e., was created). The value of this attribute is a number that represents the number of milliseconds since midnight, Jan. 1, 1970 (UTC).
+
+- Lineage Start Date (`lineageStartDate`): Any time that a FlowFile is cloned, merged, or split, this results in a "child" FlowFile being created. As those children are then cloned, merged, or split, a chain of ancestors is built. This value represents the date and time at which the oldest ancestor entered the system. Another way to think about this is that this attribute represents the latency of the FlowFile through the system. The value is a number that represents the number of millis [...]
[[process_session]]
==== ProcessSession
[nifi] 07/14: NIFI-7143: Upgrade GCP dependency
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a commit to branch support/nifi-1.11.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit 1c6863ada3fdb49b15c18f2ba61d98fc94d18752
Author: Peter Turcsanyi <tu...@apache.org>
AuthorDate: Fri Feb 14 22:15:49 2020 +0100
NIFI-7143: Upgrade GCP dependency
Signed-off-by: Pierre Villard <pi...@gmail.com>
This closes NIFI-7143.
---
nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-services-api/pom.xml | 2 +-
nifi-nar-bundles/nifi-gcp-bundle/pom.xml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-services-api/pom.xml b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-services-api/pom.xml
index 340fd40..9ae3b59 100644
--- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-services-api/pom.xml
+++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-services-api/pom.xml
@@ -33,7 +33,7 @@
<dependency>
<groupId>com.google.auth</groupId>
<artifactId>google-auth-library-oauth2-http</artifactId>
- <version>0.16.2</version>
+ <version>0.18.0</version>
<exclusions>
<exclusion>
<groupId>com.google.code.findbugs</groupId>
diff --git a/nifi-nar-bundles/nifi-gcp-bundle/pom.xml b/nifi-nar-bundles/nifi-gcp-bundle/pom.xml
index 06b83e3..2392adc 100644
--- a/nifi-nar-bundles/nifi-gcp-bundle/pom.xml
+++ b/nifi-nar-bundles/nifi-gcp-bundle/pom.xml
@@ -27,7 +27,7 @@
<packaging>pom</packaging>
<properties>
- <google.cloud.sdk.version>0.107.0-alpha</google.cloud.sdk.version>
+ <google.cloud.sdk.version>0.120.2-alpha</google.cloud.sdk.version>
</properties>
<dependencyManagement>
[nifi] 01/14: NIFI-7135 - Fix Java 11 build with
com.puppycrawl.tools:checkstyle:jar:8.29 dependency
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a commit to branch support/nifi-1.11.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit 5ed7dbce69e9895540cb611d5816684fb6ad6760
Author: Pierre Villard <pi...@gmail.com>
AuthorDate: Tue Feb 11 19:06:14 2020 -0800
NIFI-7135 - Fix Java 11 build with com.puppycrawl.tools:checkstyle:jar:8.29 dependency
This closes #4050.
Signed-off-by: Andy LoPresto <al...@apache.org>
---
.travis.yml | 21 +++++++--------------
pom.xml | 27 +++++++++++++++++++++++++++
2 files changed, 34 insertions(+), 14 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index c6e431c..a5fd764 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -28,8 +28,10 @@ matrix:
echo "Cached Maven repository exists, skipping dependency cache"
fi
- stage: "Cache"
- name: "Cache Dependencies - openjdk8"
- jdk: openjdk8
+ name: "Cache Dependencies - openjdk8 & openjdk11"
+ jdk:
+ - openjdk8
+ - openjdk11
script: >-
if [ ! -d "$HOME"/.m2/repository ] || [ ! "$(ls -A "$HOME"/.m2/repository)" ] ; then
echo "Cached Maven repository does not exist, downloading dependencies..."
@@ -38,6 +40,7 @@ matrix:
else
echo "Cached Maven repository exists, skipping dependency cache"
fi
+
# For the build stages, use -pl to exclude the following from the build to reduce build time:
# non-api nar modules
# assembly modules
@@ -74,18 +77,8 @@ matrix:
&& test ${PIPESTATUS[0]} -eq 0
- stage: "Build"
name: "Build Java 11 EN"
- # Do not specify "jdk:" here, install-jdk.sh will download the JDK set JAVA_HOME appropriately
- before_script:
- # Download the newest version of sormuras' install-jdk.sh to /tmp
- # install-jdk.sh is used by Travis internally, sormoras is the maintainer of that script
- - wget -O /tmp/install-jdk.sh https://github.com/sormuras/bach/raw/master/install-jdk.sh
- # Need to specifically install AdoptOpenJDK 11.0.4 (Linux, HotSpot) since Travis does not offer it by default
- # The link to the AdoptOpenJDK 11.0.4 .tar.gz is taken directly from AdoptOpenJDK's website
- - >-
- source /tmp/install-jdk.sh
- --url 'https://github.com/AdoptOpenJDK/openjdk11-binaries/releases/download/jdk-11.0.4%2B11/OpenJDK11U-jdk_x64_linux_hotspot_11.0.4_11.tar.gz'
- script:
- - >-
+ jdk: openjdk11
+ script: >-
mvn clean install -V -T 1C
-pl `find . -type d \( -name "*-nar" -or -name "*-assembly" -or -name "*hive3*" -or -name "nifi-system-test*" \) -and -not -name "*api-nar" -and -not -path "*/target/*" -and -not -name "*__*" -printf "!./%P,"`
-Pcontrib-check,include-grpc -Ddir-only
diff --git a/pom.xml b/pom.xml
index 1b21776..1fe9527 100644
--- a/pom.xml
+++ b/pom.xml
@@ -143,6 +143,33 @@
<pluginRepositories>
<pluginRepository>
+ <id>gcs-maven-central-mirror</id>
+ <!--
+ Google Mirror of Maven Central, placed first so that it's used instead of flaky Maven Central.
+ See https://storage-download.googleapis.com/maven-central/index.html
+ -->
+ <name>GCS Maven Central mirror</name>
+ <url>https://maven-central.storage-download.googleapis.com/repos/central/data/</url>
+ <releases>
+ <enabled>true</enabled>
+ </releases>
+ <snapshots>
+ <enabled>false</enabled>
+ </snapshots>
+ </pluginRepository>
+ <pluginRepository>
+ <id>central</id>
+ <name>Central Repository</name>
+ <url>https://repo.maven.apache.org/maven2</url>
+ <layout>default</layout>
+ <snapshots>
+ <enabled>false</enabled>
+ </snapshots>
+ <releases>
+ <updatePolicy>never</updatePolicy>
+ </releases>
+ </pluginRepository>
+ <pluginRepository>
<id>bintray</id>
<name>Groovy Bintray</name>
<url>https://dl.bintray.com/groovy/maven</url>
[nifi] 10/14: NIFI-6867: Validate PutKudu operation type property
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a commit to branch support/nifi-1.11.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit e4b905b1bc526aed88cd30cd8d934d0d24d5226a
Author: Grant Henke <gh...@cloudera.com>
AuthorDate: Fri Feb 14 10:00:00 2020 -0600
NIFI-6867: Validate PutKudu operation type property
This patch adds validation to the PutKudu operation type property.
It also improves the description to include the valid values and
adjusts the inputs to be case insensitive.
Signed-off-by: Pierre Villard <pi...@gmail.com>
This closes #4063.
---
.../org/apache/nifi/processors/kudu/PutKudu.java | 38 +++++++++++++++++++---
.../org/apache/nifi/processors/kudu/ITPutKudu.java | 5 ++-
2 files changed, 38 insertions(+), 5 deletions(-)
diff --git a/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/main/java/org/apache/nifi/processors/kudu/PutKudu.java b/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/main/java/org/apache/nifi/processors/kudu/PutKudu.java
index aedac37..3769932 100644
--- a/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/main/java/org/apache/nifi/processors/kudu/PutKudu.java
+++ b/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/main/java/org/apache/nifi/processors/kudu/PutKudu.java
@@ -38,6 +38,9 @@ import org.apache.nifi.annotation.lifecycle.OnScheduled;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.PropertyDescriptor.Builder;
import org.apache.nifi.components.PropertyValue;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.components.Validator;
import org.apache.nifi.expression.ExpressionLanguageScope;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.processor.ProcessContext;
@@ -58,6 +61,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -124,13 +128,39 @@ public class PutKudu extends AbstractKuduProcessor {
.addValidator(StandardValidators.BOOLEAN_VALIDATOR)
.build();
+ protected static final Validator OperationTypeValidator = new Validator() {
+ @Override
+ public ValidationResult validate(String subject, String value, ValidationContext context) {
+ if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(value)) {
+ return new ValidationResult.Builder().subject(subject).input(value)
+ .explanation("Expression Language Present").valid(true).build();
+ }
+
+ boolean valid;
+ try {
+ OperationType.valueOf(value.toUpperCase());
+ valid = true;
+ } catch (IllegalArgumentException ex) {
+ valid = false;
+ }
+
+ final String explanation = valid ? null :
+ "Value must be one of: " +
+ Arrays.stream(OperationType.values()).map(Enum::toString).collect(Collectors.joining(", "));
+ return new ValidationResult.Builder().subject(subject).input(value).valid(valid)
+ .explanation(explanation).build();
+ }
+ };
+
protected static final PropertyDescriptor INSERT_OPERATION = new Builder()
.name("Insert Operation")
.displayName("Kudu Operation Type")
- .description("Specify operationType for this processor. Insert-Ignore will ignore duplicated rows")
+ .description("Specify operationType for this processor.\n" +
+ "Valid values are: " +
+ Arrays.stream(OperationType.values()).map(Enum::toString).collect(Collectors.joining(", ")))
.defaultValue(OperationType.INSERT.toString())
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
- .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .addValidator(OperationTypeValidator)
.build();
protected static final PropertyDescriptor FLUSH_MODE = new Builder()
@@ -227,7 +257,7 @@ public class PutKudu extends AbstractKuduProcessor {
public void onScheduled(final ProcessContext context) throws IOException, LoginException {
batchSize = context.getProperty(BATCH_SIZE).evaluateAttributeExpressions().asInteger();
ffbatch = context.getProperty(FLOWFILE_BATCH_SIZE).evaluateAttributeExpressions().asInteger();
- flushMode = SessionConfiguration.FlushMode.valueOf(context.getProperty(FLUSH_MODE).getValue());
+ flushMode = SessionConfiguration.FlushMode.valueOf(context.getProperty(FLUSH_MODE).getValue().toUpperCase());
createKuduClient(context);
}
@@ -271,7 +301,7 @@ public class PutKudu extends AbstractKuduProcessor {
final RecordReader recordReader = recordReaderFactory.createRecordReader(flowFile, in, getLogger())) {
final String tableName = getEvaluatedProperty(TABLE_NAME, context, flowFile);
- final OperationType operationType = OperationType.valueOf(getEvaluatedProperty(INSERT_OPERATION, context, flowFile));
+ final OperationType operationType = OperationType.valueOf(getEvaluatedProperty(INSERT_OPERATION, context, flowFile).toUpperCase());
final Boolean ignoreNull = Boolean.valueOf(getEvaluatedProperty(IGNORE_NULL, context, flowFile));
final Boolean lowercaseFields = Boolean.valueOf(getEvaluatedProperty(LOWERCASE_FIELD_NAMES, context, flowFile));
final Boolean handleSchemaDrift = Boolean.valueOf(getEvaluatedProperty(HANDLE_SCHEMA_DRIFT, context, flowFile));
diff --git a/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/test/java/org/apache/nifi/processors/kudu/ITPutKudu.java b/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/test/java/org/apache/nifi/processors/kudu/ITPutKudu.java
index eee5e60..b3ad170 100644
--- a/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/test/java/org/apache/nifi/processors/kudu/ITPutKudu.java
+++ b/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/test/java/org/apache/nifi/processors/kudu/ITPutKudu.java
@@ -136,10 +136,13 @@ public class ITPutKudu {
flowFileAttributes.put(CoreAttributes.FILENAME.key(), filename);
// Use values to ensure multiple batches and multiple flow files per-trigger
- testRunner.setProperty(PutKudu.INSERT_OPERATION, OperationType.UPSERT.toString());
testRunner.setProperty(PutKudu.BATCH_SIZE, "10");
testRunner.setProperty(PutKudu.FLOWFILE_BATCH_SIZE, "2");
+ // Set the operation type.
+ flowFileAttributes.put("kudu.operation.type", "upsert");
+ testRunner.setProperty(PutKudu.INSERT_OPERATION, "${kudu.operation.type}");
+
// Don't ignore null values.
flowFileAttributes.put("kudu.ignore.null", "false");
testRunner.setProperty(PutKudu.IGNORE_NULL, "${kudu.ignore.null}");
[nifi] 12/14: NIFI-6927 Consolidate SSL context and trust managers
for OkHttp on JDK9. Fixes name conflicts.
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a commit to branch support/nifi-1.11.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit 46d3b6b0dc28f04da124be7685f82bec52e88775
Author: Troy Melhase <tr...@troy.io>
AuthorDate: Mon Feb 10 11:56:09 2020 -0900
NIFI-6927 Consolidate SSL context and trust managers for OkHttp on JDK9.
Fixes name conflicts.
This closes #4047.
Signed-off-by: Andy LoPresto <al...@apache.org>
---
.../notification/http/HttpNotificationService.java | 69 ++++------------
.../nifi/security/util/SslContextFactory.java | 71 +++++++++++++----
.../nifi-elasticsearch-processors/pom.xml | 2 +-
.../AbstractElasticsearchHttpProcessor.java | 31 +++++++-
.../TestQueryElasticsearchHttpNoHits.java | 2 +-
.../okhttp/OkHttpReplicationClient.java | 84 +++++++-------------
.../security/util/SslServerSocketFactory.java | 82 -------------------
.../framework/security/util/SslSocketFactory.java | 92 ----------------------
.../nifi/processors/standard/InvokeHTTP.java | 20 ++++-
.../org/apache/nifi/lookup/RestLookupService.java | 38 ++++++++-
10 files changed, 184 insertions(+), 307 deletions(-)
diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/notification/http/HttpNotificationService.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/notification/http/HttpNotificationService.java
index 1405625..2ab7c72 100644
--- a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/notification/http/HttpNotificationService.java
+++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/notification/http/HttpNotificationService.java
@@ -28,16 +28,13 @@ import org.apache.nifi.bootstrap.notification.NotificationFailedException;
import org.apache.nifi.bootstrap.notification.NotificationInitializationContext;
import org.apache.nifi.bootstrap.notification.NotificationType;
import org.apache.nifi.components.PropertyDescriptor;
-import org.apache.nifi.components.PropertyValue;
import org.apache.nifi.expression.AttributeExpression;
import org.apache.nifi.expression.ExpressionLanguageScope;
-import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.security.util.SslContextFactory;
import org.apache.nifi.util.Tuple;
import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLSocketFactory;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import java.util.ArrayList;
@@ -196,12 +193,25 @@ public class HttpNotificationService extends AbstractNotificationService {
// check if the keystore is set and add the factory if so
if (url.toLowerCase().startsWith("https")) {
try {
- Tuple<SSLSocketFactory, TrustManager[]> sslSocketFactoryWithTrustManagers = getSslSocketFactory(context);
+ Tuple<SSLContext, TrustManager[]> sslContextTuple = SslContextFactory.createTrustSslContextWithTrustManagers(
+ context.getProperty(HttpNotificationService.PROP_KEYSTORE).getValue(),
+ context.getProperty(HttpNotificationService.PROP_KEYSTORE_PASSWORD).isSet()
+ ? context.getProperty(HttpNotificationService.PROP_KEYSTORE_PASSWORD).getValue().toCharArray() : null,
+ context.getProperty(HttpNotificationService.PROP_KEY_PASSWORD).isSet()
+ ? context.getProperty(HttpNotificationService.PROP_KEY_PASSWORD).getValue().toCharArray() : null,
+ context.getProperty(HttpNotificationService.PROP_KEYSTORE_TYPE).getValue(),
+ context.getProperty(HttpNotificationService.PROP_TRUSTSTORE).getValue(),
+ context.getProperty(HttpNotificationService.PROP_TRUSTSTORE_PASSWORD).isSet()
+ ? context.getProperty(HttpNotificationService.PROP_TRUSTSTORE_PASSWORD).getValue().toCharArray() : null,
+ context.getProperty(HttpNotificationService.PROP_TRUSTSTORE_TYPE).getValue(),
+ SslContextFactory.ClientAuth.REQUIRED,
+ context.getProperty(HttpNotificationService.SSL_ALGORITHM).getValue()
+ );
// Find the first X509TrustManager
- List<X509TrustManager> x509TrustManagers = Arrays.stream(sslSocketFactoryWithTrustManagers.getValue())
+ List<X509TrustManager> x509TrustManagers = Arrays.stream(sslContextTuple.getValue())
.filter(trustManager -> trustManager instanceof X509TrustManager)
.map(trustManager -> (X509TrustManager) trustManager).collect(Collectors.toList());
- okHttpClientBuilder.sslSocketFactory(sslSocketFactoryWithTrustManagers.getKey(), x509TrustManagers.get(0));
+ okHttpClientBuilder.sslSocketFactory(sslContextTuple.getKey().getSocketFactory(), x509TrustManagers.get(0));
} catch (Exception e) {
throw new IllegalStateException(e);
}
@@ -249,51 +259,4 @@ public class HttpNotificationService extends AbstractNotificationService {
throw new NotificationFailedException("Failed to send Http Notification", e);
}
}
-
- private static Tuple<SSLSocketFactory, TrustManager[]> getSslSocketFactory(NotificationInitializationContext context) {
-
- final String protocol = context.getProperty(SSL_ALGORITHM).getValue();
- try {
- final PropertyValue keyPasswdProp = context.getProperty(PROP_KEY_PASSWORD);
- final char[] keyPassword = keyPasswdProp.isSet() ? keyPasswdProp.getValue().toCharArray() : null;
-
- final Tuple<SSLContext, TrustManager[]> sslContextWithTrustManagers;
- final String truststoreFile = context.getProperty(PROP_TRUSTSTORE).getValue();
- final String keystoreFile = context.getProperty(PROP_KEYSTORE).getValue();
-
- if (keystoreFile == null) {
- // If keystore not specified, create SSL Context based only on trust store.
- sslContextWithTrustManagers = SslContextFactory.createTrustSslContextWithTrustManagers(
- context.getProperty(PROP_TRUSTSTORE).getValue(),
- context.getProperty(PROP_TRUSTSTORE_PASSWORD).getValue().toCharArray(),
- context.getProperty(PROP_TRUSTSTORE_TYPE).getValue(),
- protocol);
-
- } else if (truststoreFile == null) {
- // If truststore not specified, create SSL Context based only on key store.
- sslContextWithTrustManagers = SslContextFactory.createSslContextWithTrustManagers(
- context.getProperty(PROP_KEYSTORE).getValue(),
- context.getProperty(PROP_KEYSTORE_PASSWORD).getValue().toCharArray(),
- keyPassword,
- context.getProperty(PROP_KEYSTORE_TYPE).getValue(),
- protocol);
-
- } else {
- sslContextWithTrustManagers = SslContextFactory.createSslContextWithTrustManagers(
- context.getProperty(PROP_KEYSTORE).getValue(),
- context.getProperty(PROP_KEYSTORE_PASSWORD).getValue().toCharArray(),
- keyPassword,
- context.getProperty(PROP_KEYSTORE_TYPE).getValue(),
- context.getProperty(PROP_TRUSTSTORE).getValue(),
- context.getProperty(PROP_TRUSTSTORE_PASSWORD).getValue().toCharArray(),
- context.getProperty(PROP_TRUSTSTORE_TYPE).getValue(),
- SslContextFactory.ClientAuth.REQUIRED,
- protocol);
- }
-
- return new Tuple<>(sslContextWithTrustManagers.getKey().getSocketFactory(), sslContextWithTrustManagers.getValue());
- } catch (final Exception e) {
- throw new ProcessException(e);
- }
- }
}
diff --git a/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/SslContextFactory.java b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/SslContextFactory.java
index 8863260..3e8058f 100644
--- a/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/SslContextFactory.java
+++ b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/SslContextFactory.java
@@ -50,7 +50,7 @@ public final class SslContextFactory {
}
/**
- * Creates a SSLContext instance using the given information. The password for the key is assumed to be the same
+ * Creates an SSLContext instance using the given information. The password for the key is assumed to be the same
* as the password for the keystore. If this is not the case, the {@link #createSslContext(String, char[], char[], String, String, char[], String, ClientAuth, String)}
* method should be used instead
*
@@ -63,7 +63,7 @@ public final class SslContextFactory {
* @param clientAuth the type of client authentication
* @param protocol the protocol to use for the SSL connection
*
- * @return a SSLContext instance
+ * @return an SSLContext instance
* @throws java.security.KeyStoreException if any issues accessing the keystore
* @throws java.io.IOException for any problems loading the keystores
* @throws java.security.NoSuchAlgorithmException if an algorithm is found to be used but is unknown
@@ -83,10 +83,11 @@ public final class SslContextFactory {
}
/**
- * Creates a SSLContext instance using the given information.
+ * Creates an SSLContext instance using the given information.
*
* @param keystore the full path to the keystore
* @param keystorePasswd the keystore password
+ * @param keyPasswd the password for the key within the keystore
* @param keystoreType the type of keystore (e.g., PKCS12, JKS)
* @param truststore the full path to the truststore
* @param truststorePasswd the truststore password
@@ -94,7 +95,7 @@ public final class SslContextFactory {
* @param clientAuth the type of client authentication
* @param protocol the protocol to use for the SSL connection
*
- * @return a SSLContext instance
+ * @return an SSLContext instance
* @throws java.security.KeyStoreException if any issues accessing the keystore
* @throws java.io.IOException for any problems loading the keystores
* @throws java.security.NoSuchAlgorithmException if an algorithm is found to be used but is unknown
@@ -113,10 +114,11 @@ public final class SslContextFactory {
}
/**
- * Creates a SSLContext instance paired with its TrustManager instances using the given information.
+ * Creates an SSLContext instance paired with its TrustManager instances using the given information.
*
* @param keystore the full path to the keystore
* @param keystorePasswd the keystore password
+ * @param keyPasswd the password for the key within the keystore
* @param keystoreType the type of keystore (e.g., PKCS12, JKS)
* @param truststore the full path to the truststore
* @param truststorePasswd the truststore password
@@ -124,7 +126,7 @@ public final class SslContextFactory {
* @param clientAuth the type of client authentication
* @param protocol the protocol to use for the SSL connection
*
- * @return a {@link Tuple} pairing a SSLContext instance with its TrustManagers
+ * @return a {@link Tuple} pairing an SSLContext instance with its TrustManagers
* @throws java.security.KeyStoreException if any issues accessing the keystore
* @throws java.io.IOException for any problems loading the keystores
* @throws java.security.NoSuchAlgorithmException if an algorithm is found to be used but is unknown
@@ -174,7 +176,7 @@ public final class SslContextFactory {
}
/**
- * Creates a SSLContext instance using the given information. This method assumes that the key password is
+ * Creates an SSLContext instance using the given information. This method assumes that the key password is
* the same as the keystore password. If this is not the case, use the {@link #createSslContext(String, char[], char[], String, String)}
* method instead.
*
@@ -183,7 +185,7 @@ public final class SslContextFactory {
* @param keystoreType the type of keystore (e.g., PKCS12, JKS)
* @param protocol the protocol to use for the SSL connection
*
- * @return a SSLContext instance
+ * @return an SSLContext instance
* @throws java.security.KeyStoreException if any issues accessing the keystore
* @throws java.io.IOException for any problems loading the keystores
* @throws java.security.NoSuchAlgorithmException if an algorithm is found to be used but is unknown
@@ -201,14 +203,15 @@ public final class SslContextFactory {
}
/**
- * Creates a SSLContext instance using the given information.
+ * Creates an SSLContext instance using the given information.
*
* @param keystore the full path to the keystore
* @param keystorePasswd the keystore password
+ * @param keyPasswd the password for the key within the keystore
* @param keystoreType the type of keystore (e.g., PKCS12, JKS)
* @param protocol the protocol to use for the SSL connection
*
- * @return a SSLContext instance
+ * @return an SSLContext instance
* @throws java.security.KeyStoreException if any issues accessing the keystore
* @throws java.io.IOException for any problems loading the keystores
* @throws java.security.NoSuchAlgorithmException if an algorithm is found to be used but is unknown
@@ -224,14 +227,15 @@ public final class SslContextFactory {
}
/**
- * Creates a SSLContext instance paired with its TrustManager instances using the given information.
+ * Creates an SSLContext instance paired with its TrustManager instances using the given information.
*
* @param keystore the full path to the keystore
* @param keystorePasswd the keystore password
+ * @param keyPasswd the password for the key within the keystore
* @param keystoreType the type of keystore (e.g., PKCS12, JKS)
* @param protocol the protocol to use for the SSL connection
*
- * @return a {@link Tuple} pairing a SSLContext instance paired with its TrustManager instances
+ * @return a {@link Tuple} pairing an SSLContext instance paired with its TrustManager instances
* @throws java.security.KeyStoreException if any issues accessing the keystore
* @throws java.io.IOException for any problems loading the keystores
* @throws java.security.NoSuchAlgorithmException if an algorithm is found to be used but is unknown
@@ -265,14 +269,14 @@ public final class SslContextFactory {
}
/**
- * Creates a SSLContext instance using the given information.
+ * Creates an SSLContext instance using the given information.
*
* @param truststore the full path to the truststore
* @param truststorePasswd the truststore password
* @param truststoreType the type of truststore (e.g., PKCS12, JKS)
* @param protocol the protocol to use for the SSL connection
*
- * @return a SSLContext instance
+ * @return an SSLContext instance
* @throws java.security.KeyStoreException if any issues accessing the keystore
* @throws java.io.IOException for any problems loading the keystores
* @throws java.security.NoSuchAlgorithmException if an algorithm is found to be used but is unknown
@@ -290,14 +294,14 @@ public final class SslContextFactory {
}
/**
- * Creates a SSLContext instance paired with its TrustManager instances using the given information.
+ * Creates an SSLContext instance paired with its TrustManager instances using the given information.
*
* @param truststore the full path to the truststore
* @param truststorePasswd the truststore password
* @param truststoreType the type of truststore (e.g., PKCS12, JKS)
* @param protocol the protocol to use for the SSL connection
*
- * @return a {@link Tuple} pairing a SSLContext instance paired with its TrustManager instances
+ * @return a {@link Tuple} pairing an SSLContext instance paired with its TrustManager instances
* @throws KeyStoreException if any issues accessing the keystore
* @throws IOException for any problems loading the keystores
* @throws NoSuchAlgorithmException if an algorithm is found to be used but is unknown
@@ -324,4 +328,39 @@ public final class SslContextFactory {
return new Tuple<>(ctx, trustManagers);
}
+
+ /**
+ * Creates an SSLContext instance paired with its TrustManager instances using the given information.
+ *
+ * @param keystore the full path to the keystore
+ * @param keystorePasswd the keystore password
+ * @param keyPasswd the password for the key within the keystore
+ * @param keystoreType the type of keystore (e.g., PKCS12, JKS)
+ * @param truststore the full path to the truststore
+ * @param truststorePasswd the truststore password
+ * @param truststoreType the type of truststore (e.g., PKCS12, JKS)
+ * @param clientAuth the type of client authentication
+ * @param protocol the protocol to use for the SSL connection
+ *
+ * @return a {@link Tuple} pairing an SSLSocketFactory instance with its TrustManagers
+ *
+ */
+ public static Tuple<SSLContext, TrustManager[]> createTrustSslContextWithTrustManagers(
+ final String keystore, final char[] keystorePasswd, final char[] keyPasswd, final String keystoreType,
+ final String truststore, final char[] truststorePasswd, final String truststoreType,
+ final ClientAuth clientAuth, final String protocol) throws CertificateException, UnrecoverableKeyException,
+ NoSuchAlgorithmException, KeyStoreException, KeyManagementException, IOException {
+
+ final Tuple<SSLContext, TrustManager[]> sslContextWithTrustManagers;
+ if (keystore == null) {
+ sslContextWithTrustManagers = createTrustSslContextWithTrustManagers(truststore, truststorePasswd, truststoreType, protocol);
+ } else if (truststore == null) {
+ sslContextWithTrustManagers = createSslContextWithTrustManagers(keystore, keystorePasswd, keyPasswd, keystoreType, protocol);
+ } else {
+ sslContextWithTrustManagers = createSslContextWithTrustManagers(keystore, keystorePasswd, keyPasswd, keystoreType, truststore,
+ truststorePasswd, truststoreType, clientAuth, protocol);
+ }
+ return new Tuple<>(sslContextWithTrustManagers.getKey(), sslContextWithTrustManagers.getValue());
+
+ }
}
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/pom.xml b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/pom.xml
index c14a977..de29542 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/pom.xml
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/pom.xml
@@ -95,7 +95,7 @@ language governing permissions and limitations under the License. -->
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
- <version>3.3.1</version>
+ <version>3.14.4</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/AbstractElasticsearchHttpProcessor.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/AbstractElasticsearchHttpProcessor.java
index 5787e59..49a1865 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/AbstractElasticsearchHttpProcessor.java
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/AbstractElasticsearchHttpProcessor.java
@@ -36,20 +36,31 @@ import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.proxy.ProxyConfiguration;
import org.apache.nifi.proxy.ProxySpec;
+import org.apache.nifi.security.util.SslContextFactory;
import org.apache.nifi.ssl.SSLContextService;
import org.apache.nifi.util.StringUtils;
+import org.apache.nifi.util.Tuple;
import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
import java.io.IOException;
import java.io.InputStream;
import java.net.Proxy;
import java.net.URL;
+import java.security.KeyManagementException;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.UnrecoverableKeyException;
+import java.security.cert.CertificateException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
/**
* A base class for Elasticsearch processors that use the HTTP API
@@ -212,7 +223,25 @@ public abstract class AbstractElasticsearchHttpProcessor extends AbstractElastic
// check if the ssl context is set and add the factory if so
if (sslContext != null) {
- okHttpClient.sslSocketFactory(sslContext.getSocketFactory());
+ try {
+ Tuple<SSLContext, TrustManager[]> sslContextTuple = SslContextFactory.createTrustSslContextWithTrustManagers(
+ sslService.getKeyStoreFile(),
+ sslService.getKeyStorePassword() != null ? sslService.getKeyStorePassword().toCharArray() : null,
+ sslService.getKeyPassword() != null ? sslService.getKeyPassword().toCharArray() : null,
+ sslService.getKeyStoreType(),
+ sslService.getTrustStoreFile(),
+ sslService.getTrustStorePassword() != null ? sslService.getTrustStorePassword().toCharArray() : null,
+ sslService.getTrustStoreType(),
+ SslContextFactory.ClientAuth.WANT,
+ sslService.getSslAlgorithm()
+ );
+ List<X509TrustManager> x509TrustManagers = Arrays.stream(sslContextTuple.getValue())
+ .filter(trustManager -> trustManager instanceof X509TrustManager)
+ .map(trustManager -> (X509TrustManager) trustManager).collect(Collectors.toList());
+ okHttpClient.sslSocketFactory(sslContextTuple.getKey().getSocketFactory(), x509TrustManagers.get(0));
+ } catch (CertificateException | UnrecoverableKeyException | NoSuchAlgorithmException | KeyStoreException | KeyManagementException | IOException e) {
+ throw new ProcessException(e);
+ }
}
okHttpClientAtomicReference.set(okHttpClient.build());
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/test/java/org/apache/nifi/processors/elasticsearch/TestQueryElasticsearchHttpNoHits.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/test/java/org/apache/nifi/processors/elasticsearch/TestQueryElasticsearchHttpNoHits.java
index b648ec8..6b6d906 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/test/java/org/apache/nifi/processors/elasticsearch/TestQueryElasticsearchHttpNoHits.java
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/test/java/org/apache/nifi/processors/elasticsearch/TestQueryElasticsearchHttpNoHits.java
@@ -257,7 +257,7 @@ public class TestQueryElasticsearchHttpNoHits {
if (expectHitCountOnQueryInfo) {
out.assertAttributeEquals("es.query.hitcount", String.valueOf(expectedHits));
}
- Assert.assertTrue(out.getAttribute("es.query.url").startsWith("http://127.0.0.1:9200/doc/status/_search?q=source:Twitter%20AND%20identifier:%22%22&size=2"));
+ Assert.assertTrue(out.getAttribute("es.query.url").startsWith("http://127.0.0.1:9200/doc/status/_search?q=source%3ATwitter%20AND%20identifier%3A%22%22&size=2"));
}
}
}
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/replication/okhttp/OkHttpReplicationClient.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/replication/okhttp/OkHttpReplicationClient.java
index 81229de..2352026 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/replication/okhttp/OkHttpReplicationClient.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/replication/okhttp/OkHttpReplicationClient.java
@@ -23,12 +23,16 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.module.jaxb.JaxbAnnotationIntrospector;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
-import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
-import java.security.KeyStore;
+import java.security.KeyManagementException;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.UnrecoverableKeyException;
+import java.security.cert.CertificateException;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -39,12 +43,9 @@ import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.zip.GZIPInputStream;
-import javax.net.ssl.KeyManager;
-import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSocketFactory;
import javax.net.ssl.TrustManager;
-import javax.net.ssl.TrustManagerFactory;
import javax.net.ssl.X509TrustManager;
import javax.ws.rs.HttpMethod;
import javax.ws.rs.core.MultivaluedHashMap;
@@ -70,6 +71,9 @@ import org.apache.nifi.util.Tuple;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.StreamUtils;
+// Using static imports because of the name conflict:
+import static org.apache.nifi.security.util.SslContextFactory.ClientAuth.WANT;
+import static org.apache.nifi.security.util.SslContextFactory.createTrustSslContextWithTrustManagers;
public class OkHttpReplicationClient implements HttpReplicationClient {
private static final Logger logger = LoggerFactory.getLogger(OkHttpReplicationClient.class);
@@ -103,7 +107,8 @@ public class OkHttpReplicationClient implements HttpReplicationClient {
/**
* Checks the content length header on DELETE requests to ensure it is set to '0', avoiding request timeouts on replicated requests.
- * @param method the HTTP method of the request
+ *
+ * @param method the HTTP method of the request
* @param headers the header keys and values
*/
private void checkContentLengthHeader(String method, Map<String, String> headers) {
@@ -327,57 +332,22 @@ public class OkHttpReplicationClient implements HttpReplicationClient {
}
try {
- final KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
- final TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance("X509");
-
- // initialize the KeyManager array to null and we will overwrite later if a keystore is loaded
- KeyManager[] keyManagers = null;
-
- // we will only initialize the keystore if properties have been supplied by the SSLContextService
- final String keystoreLocation = properties.getProperty(NiFiProperties.SECURITY_KEYSTORE);
- final String keystorePass = properties.getProperty(NiFiProperties.SECURITY_KEYSTORE_PASSWD);
- final String keystoreType = properties.getProperty(NiFiProperties.SECURITY_KEYSTORE_TYPE);
-
- // prepare the keystore
- final KeyStore keyStore = KeyStore.getInstance(keystoreType);
-
- try (FileInputStream keyStoreStream = new FileInputStream(keystoreLocation)) {
- keyStore.load(keyStoreStream, keystorePass.toCharArray());
- }
-
- keyManagerFactory.init(keyStore, keystorePass.toCharArray());
- keyManagers = keyManagerFactory.getKeyManagers();
-
- // we will only initialize the truststure if properties have been supplied by the SSLContextService
- // load truststore
- final String truststoreLocation = properties.getProperty(NiFiProperties.SECURITY_TRUSTSTORE);
- final String truststorePass = properties.getProperty(NiFiProperties.SECURITY_TRUSTSTORE_PASSWD);
- final String truststoreType = properties.getProperty(NiFiProperties.SECURITY_TRUSTSTORE_TYPE);
-
- KeyStore truststore = KeyStore.getInstance(truststoreType);
- truststore.load(new FileInputStream(truststoreLocation), truststorePass.toCharArray());
- trustManagerFactory.init(truststore);
-
- // TrustManagerFactory.getTrustManagers returns a trust manager for each type of trust material. Since we are getting a trust manager factory that uses "X509"
- // as it's trust management algorithm, we are able to grab the first (and thus the most preferred) and use it as our x509 Trust Manager
- //
- // https://docs.oracle.com/javase/8/docs/api/javax/net/ssl/TrustManagerFactory.html#getTrustManagers--
- final X509TrustManager x509TrustManager;
- TrustManager[] trustManagers = trustManagerFactory.getTrustManagers();
- if (trustManagers[0] != null) {
- x509TrustManager = (X509TrustManager) trustManagers[0];
- } else {
- throw new IllegalStateException("List of trust managers is null");
- }
-
- // if keystore properties were not supplied, the keyManagers array will be null
- sslContext.init(keyManagers, trustManagerFactory.getTrustManagers(), null);
-
- final SSLSocketFactory sslSocketFactory = sslContext.getSocketFactory();
- return new Tuple<>(sslSocketFactory, x509TrustManager);
- } catch (final Exception e) {
- throw new RuntimeException("Failed to create SSL Socket Factory for replicating requests across the cluster");
+ Tuple<SSLContext, TrustManager[]> sslContextTuple = createTrustSslContextWithTrustManagers(
+ properties.getProperty(NiFiProperties.SECURITY_KEYSTORE),
+ properties.getProperty(NiFiProperties.SECURITY_KEYSTORE_PASSWD) != null ? properties.getProperty(NiFiProperties.SECURITY_KEYSTORE_PASSWD).toCharArray() : null,
+ properties.getProperty(NiFiProperties.SECURITY_KEY_PASSWD) != null ? properties.getProperty(NiFiProperties.SECURITY_KEY_PASSWD).toCharArray() : null,
+ properties.getProperty(NiFiProperties.SECURITY_KEYSTORE_TYPE),
+ properties.getProperty(NiFiProperties.SECURITY_TRUSTSTORE),
+ properties.getProperty(NiFiProperties.SECURITY_TRUSTSTORE_PASSWD) != null ? properties.getProperty(NiFiProperties.SECURITY_TRUSTSTORE_PASSWD).toCharArray() : null,
+ properties.getProperty(NiFiProperties.SECURITY_TRUSTSTORE_TYPE),
+ WANT,
+ sslContext.getProtocol());
+ List<X509TrustManager> x509TrustManagers = Arrays.stream(sslContextTuple.getValue())
+ .filter(trustManager -> trustManager instanceof X509TrustManager)
+ .map(trustManager -> (X509TrustManager) trustManager).collect(Collectors.toList());
+ return new Tuple<>(sslContextTuple.getKey().getSocketFactory(), x509TrustManagers.get(0));
+ } catch (CertificateException | UnrecoverableKeyException | NoSuchAlgorithmException | KeyStoreException | KeyManagementException | IOException e) {
+ return null;
}
}
-
}
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-security/src/main/java/org/apache/nifi/framework/security/util/SslServerSocketFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-security/src/main/java/org/apache/nifi/framework/security/util/SslServerSocketFactory.java
deleted file mode 100644
index 458157c..0000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-security/src/main/java/org/apache/nifi/framework/security/util/SslServerSocketFactory.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.framework.security.util;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.ServerSocket;
-import java.security.NoSuchAlgorithmException;
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLServerSocketFactory;
-import org.apache.nifi.util.NiFiProperties;
-
-/**
- * Implements a server socket factory for creating secure server sockets based
- * on the application's configuration properties. If the properties are
- * configured for SSL (one-way or two-way), then a SSLServerSocketFactory is
- * created and used based on those properties. Otherwise, Java's default
- * SSLServerSocketFactory is used. Specifically,
- * SSLContext.getDefault().getServerSocketFactory().
- */
-public class SslServerSocketFactory extends SSLServerSocketFactory {
-
- private SSLServerSocketFactory sslServerSocketFactory;
-
- public SslServerSocketFactory(final NiFiProperties nifiProperties) {
- final SSLContext sslCtx = SslContextFactory.createSslContext(nifiProperties);
- if (sslCtx == null) {
- try {
- sslServerSocketFactory = SSLContext.getDefault().getServerSocketFactory();
- } catch (final NoSuchAlgorithmException nsae) {
- throw new SslServerSocketFactoryCreationException(nsae);
- }
- } else {
- sslServerSocketFactory = sslCtx.getServerSocketFactory();
- }
- }
-
- @Override
- public ServerSocket createServerSocket(int i, int i1, InetAddress ia) throws IOException {
- return sslServerSocketFactory.createServerSocket(i, i1, ia);
- }
-
- @Override
- public ServerSocket createServerSocket(int i, int i1) throws IOException {
- return sslServerSocketFactory.createServerSocket(i, i1);
- }
-
- @Override
- public ServerSocket createServerSocket(int i) throws IOException {
- return sslServerSocketFactory.createServerSocket(i);
- }
-
- @Override
- public ServerSocket createServerSocket() throws IOException {
- return sslServerSocketFactory.createServerSocket();
- }
-
- @Override
- public String[] getSupportedCipherSuites() {
- return sslServerSocketFactory.getSupportedCipherSuites();
- }
-
- @Override
- public String[] getDefaultCipherSuites() {
- return sslServerSocketFactory.getDefaultCipherSuites();
- }
-
-}
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-security/src/main/java/org/apache/nifi/framework/security/util/SslSocketFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-security/src/main/java/org/apache/nifi/framework/security/util/SslSocketFactory.java
deleted file mode 100644
index fa6de56..0000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-security/src/main/java/org/apache/nifi/framework/security/util/SslSocketFactory.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.framework.security.util;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.Socket;
-import java.net.UnknownHostException;
-import java.security.NoSuchAlgorithmException;
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLSocketFactory;
-import org.apache.nifi.util.NiFiProperties;
-
-/**
- * Implements a socket factory for creating secure sockets based on the
- * application's configuration properties. If the properties are configured for
- * SSL (one-way or two-way), then a SSLSocketFactory is created and used based
- * on those properties. Otherwise, Java's default SSLSocketFactory is used.
- * Specifically, SSLContext.getDefault().getSocketFactory().
- */
-public class SslSocketFactory extends SSLSocketFactory {
-
- private final SSLSocketFactory sslSocketFactory;
-
- public SslSocketFactory(final NiFiProperties nifiProperties) {
- final SSLContext sslCtx = SslContextFactory.createSslContext(nifiProperties);
- if (sslCtx == null) {
- try {
- sslSocketFactory = SSLContext.getDefault().getSocketFactory();
- } catch (final NoSuchAlgorithmException nsae) {
- throw new SslSocketFactoryCreationException(nsae);
- }
- } else {
- sslSocketFactory = sslCtx.getSocketFactory();
- }
- }
-
- @Override
- public String[] getSupportedCipherSuites() {
- return sslSocketFactory.getSupportedCipherSuites();
- }
-
- @Override
- public String[] getDefaultCipherSuites() {
- return sslSocketFactory.getDefaultCipherSuites();
- }
-
- @Override
- public Socket createSocket(Socket socket, String string, int i, boolean bln) throws IOException {
- return sslSocketFactory.createSocket(socket, string, i, bln);
- }
-
- @Override
- public Socket createSocket(InetAddress ia, int i, InetAddress ia1, int i1) throws IOException {
- return sslSocketFactory.createSocket(ia, i, ia1, i1);
- }
-
- @Override
- public Socket createSocket(InetAddress ia, int i) throws IOException {
- return sslSocketFactory.createSocket(ia, i);
- }
-
- @Override
- public Socket createSocket(String string, int i, InetAddress ia, int i1) throws IOException, UnknownHostException {
- return sslSocketFactory.createSocket(string, i, ia, i1);
- }
-
- @Override
- public Socket createSocket(String string, int i) throws IOException, UnknownHostException {
- return sslSocketFactory.createSocket(string, i);
- }
-
- @Override
- public Socket createSocket() throws IOException {
- return sslSocketFactory.createSocket();
- }
-
-}
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/InvokeHTTP.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/InvokeHTTP.java
index c344410..78cadab 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/InvokeHTTP.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/InvokeHTTP.java
@@ -55,6 +55,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import java.util.stream.Collectors;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.KeyManager;
import javax.net.ssl.KeyManagerFactory;
@@ -104,9 +105,11 @@ import org.apache.nifi.processors.standard.util.ProxyAuthenticator;
import org.apache.nifi.processors.standard.util.SoftLimitBoundedByteArrayOutputStream;
import org.apache.nifi.proxy.ProxyConfiguration;
import org.apache.nifi.proxy.ProxySpec;
+import org.apache.nifi.security.util.SslContextFactory;
import org.apache.nifi.ssl.SSLContextService;
import org.apache.nifi.ssl.SSLContextService.ClientAuth;
import org.apache.nifi.stream.io.StreamUtils;
+import org.apache.nifi.util.Tuple;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
@@ -634,8 +637,21 @@ public final class InvokeHTTP extends AbstractProcessor {
// check if the ssl context is set and add the factory if so
if (sslContext != null) {
- setSslSocketFactory(okHttpClientBuilder, sslService, sslContext, isHttpsProxy);
- }
+ Tuple<SSLContext, TrustManager[]> sslContextTuple =SslContextFactory.createTrustSslContextWithTrustManagers(
+ sslService.getKeyStoreFile(),
+ sslService.getKeyStorePassword() != null ? sslService.getKeyStorePassword().toCharArray() : null,
+ sslService.getKeyPassword() != null ? sslService.getKeyPassword().toCharArray() : null,
+ sslService.getKeyStoreType(),
+ sslService.getTrustStoreFile(),
+ sslService.getTrustStorePassword() != null ? sslService.getTrustStorePassword().toCharArray() : null,
+ sslService.getTrustStoreType(),
+ SslContextFactory.ClientAuth.NONE,
+ sslService.getSslAlgorithm());
+ List<X509TrustManager> x509TrustManagers = Arrays.stream(sslContextTuple.getValue())
+ .filter(trustManager -> trustManager instanceof X509TrustManager)
+ .map(trustManager -> (X509TrustManager) trustManager).collect(Collectors.toList());
+ okHttpClientBuilder.sslSocketFactory(sslContextTuple.getKey().getSocketFactory(), x509TrustManagers.get(0));
+ }
setAuthenticator(okHttpClientBuilder, context);
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java
index 6ecc95d..882690d 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java
@@ -40,6 +40,7 @@ import org.apache.nifi.components.Validator;
import org.apache.nifi.controller.AbstractControllerService;
import org.apache.nifi.controller.ConfigurationContext;
import org.apache.nifi.expression.ExpressionLanguageScope;
+import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.proxy.ProxyConfiguration;
import org.apache.nifi.proxy.ProxyConfigurationService;
@@ -48,6 +49,7 @@ import org.apache.nifi.record.path.FieldValue;
import org.apache.nifi.record.path.RecordPath;
import org.apache.nifi.record.path.validation.RecordPathValidator;
import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.SslContextFactory;
import org.apache.nifi.serialization.MalformedRecordException;
import org.apache.nifi.serialization.RecordReader;
import org.apache.nifi.serialization.RecordReaderFactory;
@@ -57,12 +59,20 @@ import org.apache.nifi.serialization.record.Record;
import org.apache.nifi.serialization.record.RecordSchema;
import org.apache.nifi.ssl.SSLContextService;
import org.apache.nifi.util.StringUtils;
+import org.apache.nifi.util.Tuple;
import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.Proxy;
+import java.security.KeyManagementException;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.UnrecoverableKeyException;
+import java.security.cert.CertificateException;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
@@ -231,9 +241,33 @@ public class RestLookupService extends AbstractControllerService implements Reco
}
final SSLContextService sslService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class);
- final SSLContext sslContext = sslService == null ? null : sslService.createSSLContext(SSLContextService.ClientAuth.WANT);
if (sslService != null) {
- builder.sslSocketFactory(sslContext.getSocketFactory());
+ Tuple<SSLContext, TrustManager[]> sslContextTuple = null;
+ try {
+ sslContextTuple = SslContextFactory.createTrustSslContextWithTrustManagers(
+ sslService.getKeyStoreFile(),
+ sslService.getKeyStorePassword() != null ? sslService.getKeyStorePassword().toCharArray() : null,
+ sslService.getKeyPassword() != null ? sslService.getKeyPassword().toCharArray() : null,
+ sslService.getKeyStoreType(),
+ sslService.getTrustStoreFile(),
+ sslService.getTrustStorePassword() != null ? sslService.getTrustStorePassword().toCharArray() : null,
+ sslService.getTrustStoreType(),
+ SslContextFactory.ClientAuth.WANT,
+ sslService.getSslAlgorithm()
+ );
+ } catch (CertificateException | UnrecoverableKeyException | NoSuchAlgorithmException | KeyStoreException | KeyManagementException | IOException e) {
+ throw new ProcessException(e);
+ }
+ List<X509TrustManager> x509TrustManagers = Arrays.stream(sslContextTuple.getValue())
+ .filter(trustManager -> trustManager instanceof X509TrustManager)
+ .map(trustManager -> (X509TrustManager) trustManager).collect(Collectors.toList());
+ builder.sslSocketFactory(sslContextTuple.getKey().getSocketFactory(), x509TrustManagers.get(0));
+
+ if (sslContextTuple.getValue().length > 0) {
+ builder.sslSocketFactory(sslContextTuple.getKey().getSocketFactory(), (X509TrustManager) sslContextTuple.getValue()[0]);
+ } else {
+ throw new ProcessException("Failed to create SSL socket factory with trust manager.");
+ }
}
client = builder.build();