You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by jo...@apache.org on 2020/09/17 22:09:10 UTC
[nifi] 15/15: NIFI-7799: Relogin with Kerberos on connect exception
in DBCPConnectionPool (#4519)
This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a commit to branch support/nifi-1.12.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit 94ad325442970b639e846a4f4189ad9641308181
Author: Matthew Burgess <ma...@apache.org>
AuthorDate: Thu Sep 17 13:33:54 2020 -0400
NIFI-7799: Relogin with Kerberos on connect exception in DBCPConnectionPool (#4519)
---
.../{ => nifi-kerberos-test-utils}/pom.xml | 43 ++++++-----
.../kerberos/MockKerberosCredentialsService.java | 85 ++++++++++++++++++++++
nifi-nar-bundles/nifi-extension-utils/pom.xml | 1 +
.../nifi-hive-bundle/nifi-hive3-processors/pom.xml | 6 ++
.../processors/hive/TestPutHive3Streaming.java | 67 ++---------------
.../nifi-dbcp-service/pom.xml | 19 +++++
.../org/apache/nifi/dbcp/DBCPConnectionPool.java | 9 +++
.../java/org/apache/nifi/dbcp/DBCPServiceTest.java | 38 ++++++++++
.../src/test/resources/fake.keytab | 0
9 files changed, 187 insertions(+), 81 deletions(-)
diff --git a/nifi-nar-bundles/nifi-extension-utils/pom.xml b/nifi-nar-bundles/nifi-extension-utils/nifi-kerberos-test-utils/pom.xml
similarity index 57%
copy from nifi-nar-bundles/nifi-extension-utils/pom.xml
copy to nifi-nar-bundles/nifi-extension-utils/nifi-kerberos-test-utils/pom.xml
index 2980f23..a223d03 100644
--- a/nifi-nar-bundles/nifi-extension-utils/pom.xml
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-kerberos-test-utils/pom.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0"?>
+<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
@@ -17,25 +17,28 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nifi</groupId>
- <artifactId>nifi-nar-bundles</artifactId>
- <version>1.12.1-SNAPSHOT</version>
+ <artifactId>nifi-extension-utils</artifactId>
+ <version>1.13.0-SNAPSHOT</version>
</parent>
- <packaging>pom</packaging>
- <artifactId>nifi-extension-utils</artifactId>
- <description>
- This module contains reusable utilities related to extensions that can be shared across NARs.
- </description>
- <modules>
- <module>nifi-record-utils</module>
- <module>nifi-hadoop-utils</module>
- <module>nifi-processor-utils</module>
- <module>nifi-reporting-utils</module>
- <module>nifi-syslog-utils</module>
- <module>nifi-database-utils</module>
- <module>nifi-database-test-utils</module>
- <module>nifi-service-utils</module>
- <module>nifi-prometheus-utils</module>
- </modules>
+ <artifactId>nifi-kerberos-test-utils</artifactId>
-</project>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ <version>1.13.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-utils</artifactId>
+ <version>1.13.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kerberos-credentials-service-api</artifactId>
+ <version>1.13.0-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+
+</project>
\ No newline at end of file
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-kerberos-test-utils/src/main/java/org/apache/nifi/kerberos/MockKerberosCredentialsService.java b/nifi-nar-bundles/nifi-extension-utils/nifi-kerberos-test-utils/src/main/java/org/apache/nifi/kerberos/MockKerberosCredentialsService.java
new file mode 100644
index 0000000..b7b2a01
--- /dev/null
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-kerberos-test-utils/src/main/java/org/apache/nifi/kerberos/MockKerberosCredentialsService.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kerberos;
+
+import org.apache.nifi.annotation.lifecycle.OnEnabled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.controller.AbstractControllerService;
+import org.apache.nifi.controller.ConfigurationContext;
+import org.apache.nifi.expression.ExpressionLanguageScope;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.reporting.InitializationException;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class MockKerberosCredentialsService extends AbstractControllerService implements KerberosCredentialsService {
+
+ public static String DEFAULT_KEYTAB = "src/test/resources/fake.keytab";
+ public static String DEFAULT_PRINCIPAL = "test@REALM.COM";
+
+ private volatile String keytab = DEFAULT_KEYTAB;
+ private volatile String principal = DEFAULT_PRINCIPAL;
+
+ public static final PropertyDescriptor PRINCIPAL = new PropertyDescriptor.Builder()
+ .name("Kerberos Principal")
+ .description("Kerberos principal to authenticate as. Requires nifi.kerberos.krb5.file to be set in your nifi.properties")
+ .addValidator(StandardValidators.NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
+ .required(true)
+ .build();
+
+ public static final PropertyDescriptor KEYTAB = new PropertyDescriptor.Builder()
+ .name("Kerberos Keytab")
+ .description("Kerberos keytab associated with the principal. Requires nifi.kerberos.krb5.file to be set in your nifi.properties")
+ .addValidator(StandardValidators.FILE_EXISTS_VALIDATOR)
+ .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
+ .required(true)
+ .build();
+
+ public MockKerberosCredentialsService() {
+ }
+
+ @OnEnabled
+ public void onConfigured(final ConfigurationContext context) throws InitializationException {
+ keytab = context.getProperty(KEYTAB).getValue();
+ principal = context.getProperty(PRINCIPAL).getValue();
+ }
+
+ @Override
+ public String getKeytab() {
+ return keytab;
+ }
+
+ @Override
+ public String getPrincipal() {
+ return principal;
+ }
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ final List<PropertyDescriptor> properties = new ArrayList<>(2);
+ properties.add(KEYTAB);
+ properties.add(PRINCIPAL);
+ return properties;
+ }
+
+ @Override
+ public String getIdentifier() {
+ return "kcs";
+ }
+}
\ No newline at end of file
diff --git a/nifi-nar-bundles/nifi-extension-utils/pom.xml b/nifi-nar-bundles/nifi-extension-utils/pom.xml
index 2980f23..17e0572 100644
--- a/nifi-nar-bundles/nifi-extension-utils/pom.xml
+++ b/nifi-nar-bundles/nifi-extension-utils/pom.xml
@@ -36,6 +36,7 @@
<module>nifi-database-test-utils</module>
<module>nifi-service-utils</module>
<module>nifi-prometheus-utils</module>
+ <module>nifi-kerberos-test-utils</module>
</modules>
</project>
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/pom.xml b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/pom.xml
index edd36f8..38a3234 100644
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/pom.xml
+++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/pom.xml
@@ -147,6 +147,12 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kerberos-test-utils</artifactId>
+ <version>1.13.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
<plugins>
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/java/org/apache/nifi/processors/hive/TestPutHive3Streaming.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/java/org/apache/nifi/processors/hive/TestPutHive3Streaming.java
index 05e44fb..0db3cad 100644
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/java/org/apache/nifi/processors/hive/TestPutHive3Streaming.java
+++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/java/org/apache/nifi/processors/hive/TestPutHive3Streaming.java
@@ -48,15 +48,11 @@ import org.apache.hive.streaming.StubSerializationError;
import org.apache.hive.streaming.StubStreamingIOFailure;
import org.apache.hive.streaming.StubTransactionError;
import org.apache.nifi.avro.AvroTypeUtil;
-import org.apache.nifi.components.PropertyDescriptor;
-import org.apache.nifi.components.ValidationContext;
-import org.apache.nifi.components.ValidationResult;
-import org.apache.nifi.controller.ControllerService;
-import org.apache.nifi.controller.ControllerServiceInitializationContext;
import org.apache.nifi.hadoop.SecurityUtil;
import org.apache.nifi.json.JsonRecordSetWriter;
import org.apache.nifi.json.JsonTreeReader;
import org.apache.nifi.kerberos.KerberosCredentialsService;
+import org.apache.nifi.kerberos.MockKerberosCredentialsService;
import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.reporting.InitializationException;
@@ -96,7 +92,6 @@ import java.sql.Timestamp;
import java.time.Instant;
import java.util.Arrays;
import java.util.Calendar;
-import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
@@ -293,6 +288,8 @@ public class TestPutHive3Streaming {
KerberosCredentialsService kcs = new MockKerberosCredentialsService();
runner.addControllerService("kcs", kcs);
runner.setProperty(KERBEROS_CREDENTIALS_SERVICE, "kcs");
+ runner.setProperty(kcs, MockKerberosCredentialsService.PRINCIPAL, "test");
+ runner.setProperty(kcs, MockKerberosCredentialsService.KEYTAB, "src/test/resources/core-site-security.xml");
runner.enableControllerService(kcs);
ugi = mock(UserGroupInformation.class);
when(hiveConfigurator.authenticate(eq(hiveConf), any(KerberosUser.class))).thenReturn(ugi);
@@ -313,8 +310,10 @@ public class TestPutHive3Streaming {
runner.setProperty(PutHive3Streaming.HIVE_CONFIGURATION_RESOURCES, "src/test/resources/core-site-security.xml, src/test/resources/hive-site-security.xml");
hiveConf.set(SecurityUtil.HADOOP_SECURITY_AUTHENTICATION, SecurityUtil.KERBEROS);
- KerberosCredentialsService kcs = new MockKerberosCredentialsService(null, null);
+ KerberosCredentialsService kcs = new MockKerberosCredentialsService();
runner.addControllerService("kcs", kcs);
+ runner.setProperty(kcs, MockKerberosCredentialsService.PRINCIPAL, "test");
+ runner.setProperty(kcs, MockKerberosCredentialsService.KEYTAB, "src/test/resources/core-site-security.xml");
runner.setProperty(KERBEROS_CREDENTIALS_SERVICE, "kcs");
runner.enableControllerService(kcs);
runner.assertNotValid();
@@ -1321,58 +1320,4 @@ public class TestPutHive3Streaming {
return null;
}
}
-
- private static class MockKerberosCredentialsService implements KerberosCredentialsService, ControllerService {
-
- private String keytab = "src/test/resources/fake.keytab";
- private String principal = "test@REALM.COM";
-
- public MockKerberosCredentialsService() {
- }
-
- public MockKerberosCredentialsService(String keytab, String principal) {
- this.keytab = keytab;
- this.principal = principal;
- }
-
- @Override
- public String getKeytab() {
- return keytab;
- }
-
- @Override
- public String getPrincipal() {
- return principal;
- }
-
- @Override
- public void initialize(ControllerServiceInitializationContext context) throws InitializationException {
-
- }
-
- @Override
- public Collection<ValidationResult> validate(ValidationContext context) {
- return Collections.EMPTY_LIST;
- }
-
- @Override
- public PropertyDescriptor getPropertyDescriptor(String name) {
- return null;
- }
-
- @Override
- public void onPropertyModified(PropertyDescriptor descriptor, String oldValue, String newValue) {
-
- }
-
- @Override
- public List<PropertyDescriptor> getPropertyDescriptors() {
- return null;
- }
-
- @Override
- public String getIdentifier() {
- return "kcs";
- }
- }
}
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-dbcp-service-bundle/nifi-dbcp-service/pom.xml b/nifi-nar-bundles/nifi-standard-services/nifi-dbcp-service-bundle/nifi-dbcp-service/pom.xml
index 80e0feb..83ddffc 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-dbcp-service-bundle/nifi-dbcp-service/pom.xml
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-dbcp-service-bundle/nifi-dbcp-service/pom.xml
@@ -115,8 +115,27 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-kerberos-test-utils</artifactId>
+ <version>1.13.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-all</artifactId>
</dependency>
</dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes combine.children="append">
+ <exclude>src/test/resources/fake.keytab</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
</project>
\ No newline at end of file
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-dbcp-service-bundle/nifi-dbcp-service/src/main/java/org/apache/nifi/dbcp/DBCPConnectionPool.java b/nifi-nar-bundles/nifi-standard-services/nifi-dbcp-service-bundle/nifi-dbcp-service/src/main/java/org/apache/nifi/dbcp/DBCPConnectionPool.java
index 786f9c3..de64a36 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-dbcp-service-bundle/nifi-dbcp-service/src/main/java/org/apache/nifi/dbcp/DBCPConnectionPool.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-dbcp-service-bundle/nifi-dbcp-service/src/main/java/org/apache/nifi/dbcp/DBCPConnectionPool.java
@@ -513,6 +513,15 @@ public class DBCPConnectionPool extends AbstractControllerService implements DBC
}
return con;
} catch (final SQLException e) {
+ // If using Kerberos, attempt to re-login
+ if (kerberosUser != null) {
+ try {
+ getLogger().info("Error getting connection, performing Kerberos re-login");
+ kerberosUser.login();
+ } catch (LoginException le) {
+ throw new ProcessException("Unable to authenticate Kerberos principal", le);
+ }
+ }
throw new ProcessException(e);
}
}
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-dbcp-service-bundle/nifi-dbcp-service/src/test/java/org/apache/nifi/dbcp/DBCPServiceTest.java b/nifi-nar-bundles/nifi-standard-services/nifi-dbcp-service-bundle/nifi-dbcp-service/src/test/java/org/apache/nifi/dbcp/DBCPServiceTest.java
index 6071474..373d13e 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-dbcp-service-bundle/nifi-dbcp-service/src/test/java/org/apache/nifi/dbcp/DBCPServiceTest.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-dbcp-service-bundle/nifi-dbcp-service/src/test/java/org/apache/nifi/dbcp/DBCPServiceTest.java
@@ -17,6 +17,8 @@
package org.apache.nifi.dbcp;
import org.apache.derby.drda.NetworkServerControl;
+import org.apache.nifi.kerberos.KerberosCredentialsService;
+import org.apache.nifi.kerberos.MockKerberosCredentialsService;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.reporting.InitializationException;
import org.apache.nifi.util.TestRunner;
@@ -304,6 +306,42 @@ public class DBCPServiceTest {
connection.close(); // return to pool
}
+ /**
+ * Test that we relogin to Kerberos if a ConnectException occurs during getConnection().
+ */
+ @Test(expected = ProcessException.class)
+ public void testConnectExceptionCausesKerberosRelogin() throws InitializationException, SQLException {
+ final TestRunner runner = TestRunners.newTestRunner(TestProcessor.class);
+ final DBCPConnectionPool service = new DBCPConnectionPool();
+ runner.addControllerService("test-good1", service);
+
+ final KerberosCredentialsService kerberosCredentialsService = new MockKerberosCredentialsService();
+ runner.addControllerService("kcs", kerberosCredentialsService);
+ runner.setProperty(kerberosCredentialsService, MockKerberosCredentialsService.PRINCIPAL, "bad@PRINCIPAL.COM");
+ runner.setProperty(kerberosCredentialsService, MockKerberosCredentialsService.KEYTAB, "src/test/resources/fake.keytab");
+ runner.enableControllerService(kerberosCredentialsService);
+
+ // set fake Derby database connection url
+ runner.setProperty(service, DBCPConnectionPool.DATABASE_URL, "jdbc:derby://localhost:1527/NoDB");
+ runner.setProperty(service, DBCPConnectionPool.DB_USER, "tester");
+ runner.setProperty(service, DBCPConnectionPool.DB_PASSWORD, "testerp");
+ // Use the client driver here rather than the embedded one, as it will generate a ConnectException for the test
+ runner.setProperty(service, DBCPConnectionPool.DB_DRIVERNAME, "org.apache.derby.jdbc.ClientDriver");
+ runner.setProperty(service, DBCPConnectionPool.KERBEROS_CREDENTIALS_SERVICE, "kcs");
+
+ try {
+ runner.enableControllerService(service);
+ } catch (AssertionError ae) {
+ // Ignore, this happens because it tries to do the initial Kerberos login
+ }
+
+ runner.assertValid(service);
+ final DBCPService dbcpService = (DBCPService) runner.getProcessContext().getControllerServiceLookup().getControllerService("test-good1");
+ Assert.assertNotNull(dbcpService);
+
+ dbcpService.getConnection();
+ }
+
@Rule
public ExpectedException exception = ExpectedException.none();
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-dbcp-service-bundle/nifi-dbcp-service/src/test/resources/fake.keytab b/nifi-nar-bundles/nifi-standard-services/nifi-dbcp-service-bundle/nifi-dbcp-service/src/test/resources/fake.keytab
new file mode 100644
index 0000000..e69de29