You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ma...@apache.org on 2018/10/24 17:37:23 UTC
nifi git commit: NIFI-5714 - Hive[3]ConnectionPool - Kerberos
Authentication issue/misleading
Repository: nifi
Updated Branches:
refs/heads/master d28b1172d -> 2201f7746
NIFI-5714 - Hive[3]ConnectionPool - Kerberos Authentication issue/misleading
add @Ignore on unit test...
Signed-off-by: Matthew Burgess <ma...@apache.org>
This closes #3086
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/2201f774
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/2201f774
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/2201f774
Branch: refs/heads/master
Commit: 2201f7746fd16874aefbd12d546565f5d105ab04
Parents: d28b117
Author: Pierre Villard <pi...@gmail.com>
Authored: Wed Oct 17 01:04:16 2018 +0200
Committer: Matthew Burgess <ma...@apache.org>
Committed: Wed Oct 24 13:33:30 2018 -0400
----------------------------------------------------------------------
.../nifi/dbcp/hive/HiveConnectionPool.java | 2 +
.../nifi/dbcp/hive/HiveConnectionPoolTest.java | 84 +++++++++++++++++---
.../src/test/resources/hive-site-security.xml | 4 +
.../src/test/resources/krb5.conf | 10 +++
.../nifi/dbcp/hive/Hive3ConnectionPool.java | 2 +
.../nifi/dbcp/hive/Hive3ConnectionPoolTest.java | 84 +++++++++++++++++---
.../src/test/resources/hive-site-security.xml | 4 +
.../src/test/resources/krb5.conf | 10 +++
8 files changed, 174 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/2201f774/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/dbcp/hive/HiveConnectionPool.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/dbcp/hive/HiveConnectionPool.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/dbcp/hive/HiveConnectionPool.java
index 2e40254..378799e 100644
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/dbcp/hive/HiveConnectionPool.java
+++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/dbcp/hive/HiveConnectionPool.java
@@ -306,10 +306,12 @@ public class HiveConnectionPool extends AbstractControllerService implements Hiv
}
log.info("Hive Security Enabled, logging in as principal {} with keytab {}", new Object[] {resolvedPrincipal, resolvedKeytab});
+
try {
ugi = hiveConfigurator.authenticate(hiveConfig, resolvedPrincipal, resolvedKeytab);
} catch (AuthenticationFailedException ae) {
log.error(ae.getMessage(), ae);
+ throw new InitializationException(ae);
}
getLogger().info("Successfully logged in as principal {} with keytab {}", new Object[] {resolvedPrincipal, resolvedKeytab});
http://git-wip-us.apache.org/repos/asf/nifi/blob/2201f774/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/dbcp/hive/HiveConnectionPoolTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/dbcp/hive/HiveConnectionPoolTest.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/dbcp/hive/HiveConnectionPoolTest.java
index 96dfb4f..95873b2 100644
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/dbcp/hive/HiveConnectionPoolTest.java
+++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/dbcp/hive/HiveConnectionPoolTest.java
@@ -17,42 +17,56 @@
package org.apache.nifi.dbcp.hive;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.isA;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.lang.reflect.UndeclaredThrowableException;
+import java.security.PrivilegedExceptionAction;
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.Map;
+
import org.apache.commons.dbcp.BasicDataSource;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.controller.AbstractControllerService;
+import org.apache.nifi.expression.ExpressionLanguageScope;
+import org.apache.nifi.hadoop.KerberosProperties;
import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.registry.VariableDescriptor;
+import org.apache.nifi.reporting.InitializationException;
import org.apache.nifi.util.MockConfigurationContext;
import org.apache.nifi.util.MockVariableRegistry;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Test;
-import java.io.IOException;
-import java.lang.reflect.Field;
-import java.lang.reflect.UndeclaredThrowableException;
-import java.security.PrivilegedExceptionAction;
-import java.sql.SQLException;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Matchers.isA;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
public class HiveConnectionPoolTest {
private UserGroupInformation userGroupInformation;
private HiveConnectionPool hiveConnectionPool;
private BasicDataSource basicDataSource;
private ComponentLog componentLog;
+ private KerberosProperties kerberosProperties;
+ private File krb5conf = new File("src/test/resources/krb5.conf");
@Before
public void setup() throws Exception {
+ // have to initialize this system property before anything else
+ System.setProperty("java.security.krb5.conf", krb5conf.getAbsolutePath());
+ System.setProperty("java.security.krb5.realm", "nifi.com");
+ System.setProperty("java.security.krb5.kdc", "nifi.kdc");
+
userGroupInformation = mock(UserGroupInformation.class);
basicDataSource = mock(BasicDataSource.class);
componentLog = mock(ComponentLog.class);
+ kerberosProperties = mock(KerberosProperties.class);
when(userGroupInformation.doAs(isA(PrivilegedExceptionAction.class))).thenAnswer(invocation -> {
try {
@@ -63,6 +77,19 @@ public class HiveConnectionPoolTest {
throw new UndeclaredThrowableException(e);
}
});
+
+ when(kerberosProperties.getKerberosKeytab()).thenReturn(new PropertyDescriptor.Builder()
+ .name("Kerberos Principal")
+ .addValidator(StandardValidators.ATTRIBUTE_EXPRESSION_LANGUAGE_VALIDATOR)
+ .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
+ .build());
+
+ when(kerberosProperties.getKerberosPrincipal()).thenReturn(new PropertyDescriptor.Builder()
+ .name("Kerberos Keytab")
+ .addValidator(StandardValidators.ATTRIBUTE_EXPRESSION_LANGUAGE_VALIDATOR)
+ .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
+ .build());
+
initPool();
}
@@ -80,6 +107,10 @@ public class HiveConnectionPoolTest {
Field componentLogField = AbstractControllerService.class.getDeclaredField("logger");
componentLogField.setAccessible(true);
componentLogField.set(hiveConnectionPool, componentLog);
+
+ Field kerberosPropertiesField = HiveConnectionPool.class.getDeclaredField("kerberosProperties");
+ kerberosPropertiesField.setAccessible(true);
+ kerberosPropertiesField.set(hiveConnectionPool, kerberosProperties);
}
@Test(expected = ProcessException.class)
@@ -135,4 +166,31 @@ public class HiveConnectionPoolTest {
assertEquals(10000L, basicDataSource.getMaxWait());
assertEquals(URL, hiveConnectionPool.getConnectionURL());
}
+
+ @Ignore("Kerberos does not seem to be properly handled in Travis build, but, locally, this test should successfully run")
+ @Test(expected = InitializationException.class)
+ public void testKerberosAuthException() throws Exception {
+ final String URL = "jdbc:hive2://localhost:10000/default";
+ final String conf = "src/test/resources/hive-site-security.xml";
+ final String ktab = "src/test/resources/fake.keytab";
+ final String kprinc = "bad@PRINCIPAL.COM";
+
+ KerberosProperties kerbProperties = new KerberosProperties(krb5conf);
+
+ Map<PropertyDescriptor, String> props = new HashMap<PropertyDescriptor, String>() {{
+ put(HiveConnectionPool.DATABASE_URL, "${url}");
+ put(HiveConnectionPool.HIVE_CONFIGURATION_RESOURCES, "${conf}");
+ put(kerbProperties.getKerberosKeytab(), "${ktab}");
+ put(kerbProperties.getKerberosPrincipal(), "${kprinc}");
+ }};
+
+ MockVariableRegistry registry = new MockVariableRegistry();
+ registry.setVariable(new VariableDescriptor("url"), URL);
+ registry.setVariable(new VariableDescriptor("conf"), conf);
+ registry.setVariable(new VariableDescriptor("ktab"), ktab);
+ registry.setVariable(new VariableDescriptor("kprinc"), kprinc);
+
+ MockConfigurationContext context = new MockConfigurationContext(props, null, registry);
+ hiveConnectionPool.onConfigured(context);
+ }
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/2201f774/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/resources/hive-site-security.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/resources/hive-site-security.xml b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/resources/hive-site-security.xml
index 07fd74c..4d64c95 100644
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/resources/hive-site-security.xml
+++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/resources/hive-site-security.xml
@@ -23,4 +23,8 @@
<name>hive.server2.authentication</name>
<value>KERBEROS</value>
</property>
+ <property>
+ <name>hadoop.security.authentication</name>
+ <value>kerberos</value>
+ </property>
</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/2201f774/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/resources/krb5.conf
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/resources/krb5.conf b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/resources/krb5.conf
index e69de29..323da39 100644
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/resources/krb5.conf
+++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/resources/krb5.conf
@@ -0,0 +1,10 @@
+[libdefaults]
+ default_realm = EXAMPLE.COM
+ dns_lookup_kdc = false
+ dns_lookup_realm = false
+
+[realms]
+ EXAMPLE.COM = {
+ kdc = kerberos.example.com
+ admin_server = kerberos.example.com
+ }
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/2201f774/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/main/java/org/apache/nifi/dbcp/hive/Hive3ConnectionPool.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/main/java/org/apache/nifi/dbcp/hive/Hive3ConnectionPool.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/main/java/org/apache/nifi/dbcp/hive/Hive3ConnectionPool.java
index b0662b8..c2042bb 100644
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/main/java/org/apache/nifi/dbcp/hive/Hive3ConnectionPool.java
+++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/main/java/org/apache/nifi/dbcp/hive/Hive3ConnectionPool.java
@@ -304,10 +304,12 @@ public class Hive3ConnectionPool extends AbstractControllerService implements Hi
}
log.info("Hive Security Enabled, logging in as principal {} with keytab {}", new Object[] {resolvedPrincipal, resolvedKeytab});
+
try {
ugi = hiveConfigurator.authenticate(hiveConfig, resolvedPrincipal, resolvedKeytab);
} catch (AuthenticationFailedException ae) {
log.error(ae.getMessage(), ae);
+ throw new InitializationException(ae);
}
getLogger().info("Successfully logged in as principal {} with keytab {}", new Object[] {resolvedPrincipal, resolvedKeytab});
http://git-wip-us.apache.org/repos/asf/nifi/blob/2201f774/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/java/org/apache/nifi/dbcp/hive/Hive3ConnectionPoolTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/java/org/apache/nifi/dbcp/hive/Hive3ConnectionPoolTest.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/java/org/apache/nifi/dbcp/hive/Hive3ConnectionPoolTest.java
index 5d9f87c..b38f41e 100644
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/java/org/apache/nifi/dbcp/hive/Hive3ConnectionPoolTest.java
+++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/java/org/apache/nifi/dbcp/hive/Hive3ConnectionPoolTest.java
@@ -17,42 +17,56 @@
package org.apache.nifi.dbcp.hive;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.isA;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.lang.reflect.UndeclaredThrowableException;
+import java.security.PrivilegedExceptionAction;
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.Map;
+
import org.apache.commons.dbcp.BasicDataSource;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.controller.AbstractControllerService;
+import org.apache.nifi.expression.ExpressionLanguageScope;
+import org.apache.nifi.hadoop.KerberosProperties;
import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.registry.VariableDescriptor;
+import org.apache.nifi.reporting.InitializationException;
import org.apache.nifi.util.MockConfigurationContext;
import org.apache.nifi.util.MockVariableRegistry;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Test;
-import java.io.IOException;
-import java.lang.reflect.Field;
-import java.lang.reflect.UndeclaredThrowableException;
-import java.security.PrivilegedExceptionAction;
-import java.sql.SQLException;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Matchers.isA;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
public class Hive3ConnectionPoolTest {
private UserGroupInformation userGroupInformation;
private Hive3ConnectionPool hive3ConnectionPool;
private BasicDataSource basicDataSource;
private ComponentLog componentLog;
+ private KerberosProperties kerberosProperties;
+ private File krb5conf = new File("src/test/resources/krb5.conf");
@Before
public void setup() throws Exception {
+ // have to initialize this system property before anything else
+ System.setProperty("java.security.krb5.conf", krb5conf.getAbsolutePath());
+ System.setProperty("java.security.krb5.realm", "nifi.com");
+ System.setProperty("java.security.krb5.kdc", "nifi.kdc");
+
userGroupInformation = mock(UserGroupInformation.class);
basicDataSource = mock(BasicDataSource.class);
componentLog = mock(ComponentLog.class);
+ kerberosProperties = mock(KerberosProperties.class);
when(userGroupInformation.doAs(isA(PrivilegedExceptionAction.class))).thenAnswer(invocation -> {
try {
@@ -63,6 +77,19 @@ public class Hive3ConnectionPoolTest {
throw new UndeclaredThrowableException(e);
}
});
+
+ when(kerberosProperties.getKerberosKeytab()).thenReturn(new PropertyDescriptor.Builder()
+ .name("Kerberos Principal")
+ .addValidator(StandardValidators.ATTRIBUTE_EXPRESSION_LANGUAGE_VALIDATOR)
+ .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
+ .build());
+
+ when(kerberosProperties.getKerberosPrincipal()).thenReturn(new PropertyDescriptor.Builder()
+ .name("Kerberos Keytab")
+ .addValidator(StandardValidators.ATTRIBUTE_EXPRESSION_LANGUAGE_VALIDATOR)
+ .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
+ .build());
+
initPool();
}
@@ -80,6 +107,10 @@ public class Hive3ConnectionPoolTest {
Field componentLogField = AbstractControllerService.class.getDeclaredField("logger");
componentLogField.setAccessible(true);
componentLogField.set(hive3ConnectionPool, componentLog);
+
+ Field kerberosPropertiesField = Hive3ConnectionPool.class.getDeclaredField("kerberosProperties");
+ kerberosPropertiesField.setAccessible(true);
+ kerberosPropertiesField.set(hive3ConnectionPool, kerberosProperties);
}
@Test(expected = ProcessException.class)
@@ -135,4 +166,31 @@ public class Hive3ConnectionPoolTest {
assertEquals(10000L, basicDataSource.getMaxWait());
assertEquals(URL, hive3ConnectionPool.getConnectionURL());
}
+
+ @Ignore("Kerberos does not seem to be properly handled in Travis build, but, locally, this test should successfully run")
+ @Test(expected = InitializationException.class)
+ public void testKerberosAuthException() throws Exception {
+ final String URL = "jdbc:hive2://localhost:10000/default";
+ final String conf = "src/test/resources/hive-site-security.xml";
+ final String ktab = "src/test/resources/fake.keytab";
+ final String kprinc = "bad@PRINCIPAL.COM";
+
+ KerberosProperties kerbProperties = new KerberosProperties(krb5conf);
+
+ Map<PropertyDescriptor, String> props = new HashMap<PropertyDescriptor, String>() {{
+ put(Hive3ConnectionPool.DATABASE_URL, "${url}");
+ put(Hive3ConnectionPool.HIVE_CONFIGURATION_RESOURCES, "${conf}");
+ put(kerbProperties.getKerberosKeytab(), "${ktab}");
+ put(kerbProperties.getKerberosPrincipal(), "${kprinc}");
+ }};
+
+ MockVariableRegistry registry = new MockVariableRegistry();
+ registry.setVariable(new VariableDescriptor("url"), URL);
+ registry.setVariable(new VariableDescriptor("conf"), conf);
+ registry.setVariable(new VariableDescriptor("ktab"), ktab);
+ registry.setVariable(new VariableDescriptor("kprinc"), kprinc);
+
+ MockConfigurationContext context = new MockConfigurationContext(props, null, registry);
+ hive3ConnectionPool.onConfigured(context);
+ }
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/2201f774/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/resources/hive-site-security.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/resources/hive-site-security.xml b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/resources/hive-site-security.xml
index 07fd74c..4d64c95 100644
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/resources/hive-site-security.xml
+++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/resources/hive-site-security.xml
@@ -23,4 +23,8 @@
<name>hive.server2.authentication</name>
<value>KERBEROS</value>
</property>
+ <property>
+ <name>hadoop.security.authentication</name>
+ <value>kerberos</value>
+ </property>
</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/2201f774/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/resources/krb5.conf
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/resources/krb5.conf b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/resources/krb5.conf
index e69de29..323da39 100644
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/resources/krb5.conf
+++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/resources/krb5.conf
@@ -0,0 +1,10 @@
+[libdefaults]
+ default_realm = EXAMPLE.COM
+ dns_lookup_kdc = false
+ dns_lookup_realm = false
+
+[realms]
+ EXAMPLE.COM = {
+ kdc = kerberos.example.com
+ admin_server = kerberos.example.com
+ }
\ No newline at end of file