You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by sr...@apache.org on 2019/07/04 18:35:54 UTC
[storm] branch master updated: STORM-3449: autocreds: fix all
checkstyle warnings
This is an automated email from the ASF dual-hosted git repository.
srdo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/storm.git
The following commit(s) were added to refs/heads/master by this push:
new f5aecc1 STORM-3449: autocreds: fix all checkstyle warnings
new 7ca1ffb Merge pull request #3065 from krichter722/checkstyle-autocreds
f5aecc1 is described below
commit f5aecc1b5d5e15c0c128f600c5a399eb0b0e8fc4
Author: Karl-Philipp Richter <kr...@posteo.de>
AuthorDate: Mon Jul 1 21:54:37 2019 +0200
STORM-3449: autocreds: fix all checkstyle warnings
---
external/storm-autocreds/pom.xml | 2 +-
.../storm/common/AbstractHadoopAutoCreds.java | 29 +++----
.../AbstractHadoopNimbusPluginAutoCreds.java | 25 +++---
.../apache/storm/common/CredentialKeyProvider.java | 13 +--
.../apache/storm/common/HadoopCredentialUtil.java | 85 ++++++++++---------
.../org/apache/storm/hbase/security/AutoHBase.java | 4 +-
.../storm/hbase/security/AutoHBaseCommand.java | 71 ++++++++--------
.../storm/hbase/security/AutoHBaseNimbus.java | 34 ++++----
.../storm/hbase/security/HBaseSecurityUtil.java | 14 ++--
.../org/apache/storm/hdfs/security/AutoHDFS.java | 5 +-
.../storm/hdfs/security/AutoHDFSCommand.java | 58 ++++++-------
.../apache/storm/hdfs/security/AutoHDFSNimbus.java | 57 +++++++------
.../storm/hdfs/security/HdfsSecurityUtil.java | 24 +++---
.../org/apache/storm/hive/security/AutoHive.java | 6 +-
.../storm/hive/security/AutoHiveCommand.java | 62 +++++++-------
.../apache/storm/hive/security/AutoHiveNimbus.java | 97 ++++++++++++----------
.../storm/hive/security/HiveSecurityUtil.java | 13 +--
17 files changed, 312 insertions(+), 287 deletions(-)
diff --git a/external/storm-autocreds/pom.xml b/external/storm-autocreds/pom.xml
index a8ac75e..87b2a2f 100644
--- a/external/storm-autocreds/pom.xml
+++ b/external/storm-autocreds/pom.xml
@@ -207,7 +207,7 @@
<artifactId>maven-checkstyle-plugin</artifactId>
<!--Note - the version would be inherited-->
<configuration>
- <maxAllowedViolations>249</maxAllowedViolations>
+ <maxAllowedViolations>0</maxAllowedViolations>
</configuration>
</plugin>
<plugin>
diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/common/AbstractHadoopAutoCreds.java b/external/storm-autocreds/src/main/java/org/apache/storm/common/AbstractHadoopAutoCreds.java
index 569f5af..7800401 100644
--- a/external/storm-autocreds/src/main/java/org/apache/storm/common/AbstractHadoopAutoCreds.java
+++ b/external/storm-autocreds/src/main/java/org/apache/storm/common/AbstractHadoopAutoCreds.java
@@ -15,8 +15,18 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.common;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import javax.security.auth.Subject;
+import javax.xml.bind.DatatypeConverter;
+
import org.apache.commons.lang.StringUtils;
import org.apache.commons.math3.util.Pair;
import org.apache.hadoop.security.Credentials;
@@ -27,15 +37,6 @@ import org.apache.storm.security.auth.IAutoCredentials;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.security.auth.Subject;
-import javax.xml.bind.DatatypeConverter;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
/**
* The base class that for auto credential plugins that abstracts out some of the common functionality.
*/
@@ -62,7 +63,7 @@ public abstract class AbstractHadoopAutoCreds implements IAutoCredentials, Crede
@Override
public void populateSubject(Subject subject, Map<String, String> credentials) {
addCredentialToSubject(subject, credentials);
- addTokensToUGI(subject);
+ addTokensToUgi(subject);
}
/**
@@ -71,7 +72,7 @@ public abstract class AbstractHadoopAutoCreds implements IAutoCredentials, Crede
@Override
public void updateSubject(Subject subject, Map<String, String> credentials) {
addCredentialToSubject(subject, credentials);
- addTokensToUGI(subject);
+ addTokensToUgi(subject);
}
public Set<Pair<String, Credentials>> getCredentials(Map<String, String> credentials) {
@@ -79,14 +80,14 @@ public abstract class AbstractHadoopAutoCreds implements IAutoCredentials, Crede
}
/**
- * Prepare the plugin
+ * Prepare the plugin.
*
* @param topoConf the topology conf
*/
protected abstract void doPrepare(Map<String, Object> topoConf);
/**
- * The lookup key for the config key string
+ * The lookup key for the config key string.
*
* @return the config key string
*/
@@ -104,7 +105,7 @@ public abstract class AbstractHadoopAutoCreds implements IAutoCredentials, Crede
}
}
- private void addTokensToUGI(Subject subject) {
+ private void addTokensToUgi(Subject subject) {
if (subject != null) {
Set<Credentials> privateCredentials = subject.getPrivateCredentials(Credentials.class);
if (privateCredentials != null) {
diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/common/AbstractHadoopNimbusPluginAutoCreds.java b/external/storm-autocreds/src/main/java/org/apache/storm/common/AbstractHadoopNimbusPluginAutoCreds.java
index 0ddf381..807d2ad 100644
--- a/external/storm-autocreds/src/main/java/org/apache/storm/common/AbstractHadoopNimbusPluginAutoCreds.java
+++ b/external/storm-autocreds/src/main/java/org/apache/storm/common/AbstractHadoopNimbusPluginAutoCreds.java
@@ -15,8 +15,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.common;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import javax.xml.bind.DatatypeConverter;
+
import org.apache.commons.lang.StringUtils;
import org.apache.commons.math3.util.Pair;
import org.apache.hadoop.conf.Configuration;
@@ -29,14 +38,6 @@ import org.apache.storm.utils.ConfigUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.xml.bind.DatatypeConverter;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
/**
* The base class that for auto credential plugins that abstracts out some of the common functionality.
*/
@@ -51,7 +52,9 @@ public abstract class AbstractHadoopNimbusPluginAutoCreds
}
@Override
- public void populateCredentials(Map<String, String> credentials, Map<String, Object> topologyConf, final String topologyOwnerPrincipal) {
+ public void populateCredentials(Map<String, String> credentials,
+ Map<String, Object> topologyConf,
+ final String topologyOwnerPrincipal) {
try {
List<String> configKeys = getConfigKeys(topologyConf);
if (!configKeys.isEmpty()) {
@@ -103,14 +106,14 @@ public abstract class AbstractHadoopNimbusPluginAutoCreds
}
/**
- * Prepare the plugin
+ * Prepare the plugin.
*
* @param conf the storm cluster conf set via storm.yaml
*/
protected abstract void doPrepare(Map<String, Object> conf);
/**
- * The lookup key for the config key string
+ * The lookup key for the config key string.
*
* @return the config key string
*/
diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/common/CredentialKeyProvider.java b/external/storm-autocreds/src/main/java/org/apache/storm/common/CredentialKeyProvider.java
index 3826407..a3503a5 100644
--- a/external/storm-autocreds/src/main/java/org/apache/storm/common/CredentialKeyProvider.java
+++ b/external/storm-autocreds/src/main/java/org/apache/storm/common/CredentialKeyProvider.java
@@ -15,16 +15,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.common;
/**
* Provider interface for credential key.
*/
public interface CredentialKeyProvider {
- /**
- * The lookup key for the config key string
- *
- * @return the config key string
- */
- String getCredentialKey(String configKey);
+ /**
+ * The lookup key for the config key string.
+ *
+ * @return the config key string
+ */
+ String getCredentialKey(String configKey);
}
diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/common/HadoopCredentialUtil.java b/external/storm-autocreds/src/main/java/org/apache/storm/common/HadoopCredentialUtil.java
index f676abf..690c391 100644
--- a/external/storm-autocreds/src/main/java/org/apache/storm/common/HadoopCredentialUtil.java
+++ b/external/storm-autocreds/src/main/java/org/apache/storm/common/HadoopCredentialUtil.java
@@ -15,66 +15,69 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.storm.common;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.math3.util.Pair;
-import org.apache.hadoop.security.Credentials;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+package org.apache.storm.common;
-import javax.xml.bind.DatatypeConverter;
import java.io.ByteArrayInputStream;
import java.io.ObjectInputStream;
import java.util.Collection;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import javax.xml.bind.DatatypeConverter;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.math3.util.Pair;
+import org.apache.hadoop.security.Credentials;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Utility class for getting credential for Hadoop.
*/
final class HadoopCredentialUtil {
- private static final Logger LOG = LoggerFactory.getLogger(HadoopCredentialUtil.class);
+ private static final Logger LOG = LoggerFactory.getLogger(HadoopCredentialUtil.class);
- private HadoopCredentialUtil() {
- }
+ private HadoopCredentialUtil() {
+ }
- static Set<Pair<String, Credentials>> getCredential(CredentialKeyProvider provider,
- Map<String, String> credentials, Collection<String> configKeys) {
- Set<Pair<String, Credentials>> res = new HashSet<>();
- if (!configKeys.isEmpty()) {
- for (String configKey : configKeys) {
- Credentials cred = doGetCredentials(provider, credentials, configKey);
- if (cred != null) {
- res.add(new Pair(configKey, cred));
+ static Set<Pair<String, Credentials>> getCredential(CredentialKeyProvider provider,
+ Map<String, String> credentials,
+ Collection<String> configKeys) {
+ Set<Pair<String, Credentials>> res = new HashSet<>();
+ if (!configKeys.isEmpty()) {
+ for (String configKey : configKeys) {
+ Credentials cred = doGetCredentials(provider, credentials, configKey);
+ if (cred != null) {
+ res.add(new Pair(configKey, cred));
+ }
+ }
+ } else {
+ Credentials cred = doGetCredentials(provider, credentials, StringUtils.EMPTY);
+ if (cred != null) {
+ res.add(new Pair(StringUtils.EMPTY, cred));
+ }
}
- }
- } else {
- Credentials cred = doGetCredentials(provider, credentials, StringUtils.EMPTY);
- if (cred != null) {
- res.add(new Pair(StringUtils.EMPTY, cred));
- }
+ return res;
}
- return res;
- }
- private static Credentials doGetCredentials(CredentialKeyProvider provider,
- Map<String, String> credentials, String configKey) {
- Credentials credential = null;
- String credentialKey = provider.getCredentialKey(configKey);
- if (credentials != null && credentials.containsKey(credentialKey)) {
- try {
- byte[] credBytes = DatatypeConverter.parseBase64Binary(credentialKey);
- ObjectInputStream in = new ObjectInputStream(new ByteArrayInputStream(credBytes));
+ private static Credentials doGetCredentials(CredentialKeyProvider provider,
+ Map<String, String> credentials,
+ String configKey) {
+ Credentials credential = null;
+ String credentialKey = provider.getCredentialKey(configKey);
+ if (credentials != null && credentials.containsKey(credentialKey)) {
+ try {
+ byte[] credBytes = DatatypeConverter.parseBase64Binary(credentialKey);
+ ObjectInputStream in = new ObjectInputStream(new ByteArrayInputStream(credBytes));
- credential = new Credentials();
- credential.readFields(in);
- } catch (Exception e) {
- LOG.error("Could not obtain credentials from credentials map.", e);
- }
+ credential = new Credentials();
+ credential.readFields(in);
+ } catch (Exception e) {
+ LOG.error("Could not obtain credentials from credentials map.", e);
+ }
+ }
+ return credential;
}
- return credential;
- }
}
diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBase.java b/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBase.java
index e38a54a..549f2d9 100644
--- a/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBase.java
+++ b/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBase.java
@@ -18,11 +18,11 @@
package org.apache.storm.hbase.security;
-import org.apache.storm.common.AbstractHadoopAutoCreds;
+import static org.apache.storm.hbase.security.HBaseSecurityUtil.HBASE_CREDENTIALS;
import java.util.Map;
-import static org.apache.storm.hbase.security.HBaseSecurityUtil.HBASE_CREDENTIALS;
+import org.apache.storm.common.AbstractHadoopAutoCreds;
/**
* Auto credentials plugin for HBase implementation. This class provides a way to automatically
diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBaseCommand.java b/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBaseCommand.java
index bd03ad9..189d32a 100644
--- a/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBaseCommand.java
+++ b/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBaseCommand.java
@@ -15,49 +15,50 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.hbase.security;
-import org.apache.storm.Config;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import static org.apache.storm.hbase.security.HBaseSecurityUtil.HBASE_KEYTAB_FILE_KEY;
+import static org.apache.storm.hbase.security.HBaseSecurityUtil.HBASE_PRINCIPAL_KEY;
-import javax.security.auth.Subject;
import java.util.HashMap;
import java.util.Map;
+import javax.security.auth.Subject;
-import static org.apache.storm.hbase.security.HBaseSecurityUtil.HBASE_KEYTAB_FILE_KEY;
-import static org.apache.storm.hbase.security.HBaseSecurityUtil.HBASE_PRINCIPAL_KEY;
+import org.apache.storm.Config;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
- * Command tool of Hive credential renewer
+ * Command tool of Hive credential renewer.
*/
public final class AutoHBaseCommand {
- private static final Logger LOG = LoggerFactory.getLogger(AutoHBaseCommand.class);
-
- private AutoHBaseCommand() {
- }
-
- @SuppressWarnings("unchecked")
- public static void main(String[] args) throws Exception {
- Map<String, Object> conf = new HashMap<>();
- conf.put(HBASE_PRINCIPAL_KEY, args[1]); // hbase principal storm-hbase@WITZEN.COM
- conf.put(HBASE_KEYTAB_FILE_KEY,
- args[2]); // storm hbase keytab /etc/security/keytabs/storm-hbase.keytab
-
- AutoHBase autoHBase = new AutoHBase();
- autoHBase.prepare(conf);
- AutoHBaseNimbus autoHBaseNimbus = new AutoHBaseNimbus();
- autoHBaseNimbus.prepare(conf);
-
- Map<String, String> creds = new HashMap<>();
- autoHBaseNimbus.populateCredentials(creds, conf, args[0]); //with realm e.g. storm@WITZEND.COM
- LOG.info("Got HBase credentials" + autoHBase.getCredentials(creds));
-
- Subject s = new Subject();
- autoHBase.populateSubject(s, creds);
- LOG.info("Got a Subject " + s);
-
- autoHBaseNimbus.renew(creds, conf, args[0]);
- LOG.info("renewed credentials" + autoHBase.getCredentials(creds));
- }
+ private static final Logger LOG = LoggerFactory.getLogger(AutoHBaseCommand.class);
+
+ private AutoHBaseCommand() {
+ }
+
+ @SuppressWarnings("unchecked")
+ public static void main(String[] args) throws Exception {
+ Map<String, Object> conf = new HashMap<>();
+ conf.put(HBASE_PRINCIPAL_KEY, args[1]); // hbase principal storm-hbase@WITZEN.COM
+ conf.put(HBASE_KEYTAB_FILE_KEY,
+ args[2]); // storm hbase keytab /etc/security/keytabs/storm-hbase.keytab
+
+ AutoHBase autoHBase = new AutoHBase();
+ autoHBase.prepare(conf);
+ AutoHBaseNimbus autoHBaseNimbus = new AutoHBaseNimbus();
+ autoHBaseNimbus.prepare(conf);
+
+ Map<String, String> creds = new HashMap<>();
+ autoHBaseNimbus.populateCredentials(creds, conf, args[0]); //with realm e.g. storm@WITZEND.COM
+ LOG.info("Got HBase credentials" + autoHBase.getCredentials(creds));
+
+ Subject s = new Subject();
+ autoHBase.populateSubject(s, creds);
+ LOG.info("Got a Subject " + s);
+
+ autoHBaseNimbus.renew(creds, conf, args[0]);
+ LOG.info("renewed credentials" + autoHBase.getCredentials(creds));
+ }
}
diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBaseNimbus.java b/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBaseNimbus.java
index c9361b1..2ad5819 100644
--- a/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBaseNimbus.java
+++ b/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBaseNimbus.java
@@ -18,6 +18,15 @@
package org.apache.storm.hbase.security;
+import static org.apache.storm.hbase.security.HBaseSecurityUtil.HBASE_CREDENTIALS;
+import static org.apache.storm.hbase.security.HBaseSecurityUtil.HBASE_KEYTAB_FILE_KEY;
+import static org.apache.storm.hbase.security.HBaseSecurityUtil.HBASE_PRINCIPAL_KEY;
+
+import java.io.ByteArrayOutputStream;
+import java.io.ObjectOutputStream;
+import java.net.InetAddress;
+import java.util.Map;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Connection;
@@ -34,15 +43,6 @@ import org.apache.storm.common.AbstractHadoopNimbusPluginAutoCreds;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.ByteArrayOutputStream;
-import java.io.ObjectOutputStream;
-import java.net.InetAddress;
-import java.util.Map;
-
-import static org.apache.storm.hbase.security.HBaseSecurityUtil.HBASE_CREDENTIALS;
-import static org.apache.storm.hbase.security.HBaseSecurityUtil.HBASE_KEYTAB_FILE_KEY;
-import static org.apache.storm.hbase.security.HBaseSecurityUtil.HBASE_PRINCIPAL_KEY;
-
/**
* Auto credentials nimbus plugin for HBase implementation. This class automatically
* gets HBase delegation tokens and push it to user's topology.
@@ -76,16 +76,10 @@ public class AutoHBaseNimbus extends AbstractHadoopNimbusPluginAutoCreds {
return getHadoopCredentials(conf, HBaseConfiguration.create(), topologyOwnerPrincipal);
}
- private Configuration getHadoopConfiguration(Map<String, Object> topoConf, String configKey) {
- Configuration configuration = HBaseConfiguration.create();
- fillHadoopConfiguration(topoConf, configKey, configuration);
- return configuration;
- }
-
@SuppressWarnings("unchecked")
protected byte[] getHadoopCredentials(Map<String, Object> conf, Configuration hbaseConf, final String topologySubmitterUser) {
try {
- if(UserGroupInformation.isSecurityEnabled()) {
+ if (UserGroupInformation.isSecurityEnabled()) {
UserProvider provider = UserProvider.instantiate(hbaseConf);
provider.login(HBASE_KEYTAB_FILE_KEY, HBASE_PRINCIPAL_KEY, InetAddress.getLocalHost().getCanonicalHostName());
@@ -97,7 +91,7 @@ public class AutoHBaseNimbus extends AbstractHadoopNimbusPluginAutoCreds {
User user = User.create(proxyUser);
- if(user.isHBaseSecurityEnabled(hbaseConf)) {
+ if (user.isHBaseSecurityEnabled(hbaseConf)) {
final Connection connection = ConnectionFactory.createConnection(hbaseConf, user);
TokenUtil.obtainAndCacheToken(connection, user);
@@ -127,6 +121,12 @@ public class AutoHBaseNimbus extends AbstractHadoopNimbusPluginAutoCreds {
}
}
+ private Configuration getHadoopConfiguration(Map<String, Object> topoConf, String configKey) {
+ Configuration configuration = HBaseConfiguration.create();
+ fillHadoopConfiguration(topoConf, configKey, configuration);
+ return configuration;
+ }
+
@Override
public void doRenew(Map<String, String> credentials, Map<String, Object> topologyConf, final String topologySubmitterUser) {
//HBASE tokens are not renewable so we always have to get new ones.
diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/HBaseSecurityUtil.java b/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/HBaseSecurityUtil.java
index cb8329b..1d8e5b8 100644
--- a/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/HBaseSecurityUtil.java
+++ b/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/HBaseSecurityUtil.java
@@ -18,19 +18,19 @@
package org.apache.storm.hbase.security;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.security.UserProvider;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.storm.security.auth.kerberos.AutoTGT;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import static org.apache.storm.Config.TOPOLOGY_AUTO_CREDENTIALS;
import java.io.IOException;
import java.net.InetAddress;
import java.util.List;
import java.util.Map;
-import static org.apache.storm.Config.TOPOLOGY_AUTO_CREDENTIALS;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.security.UserProvider;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.storm.security.auth.kerberos.AutoTGT;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* This class provides util methods for storm-hbase connector communicating
diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFS.java b/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFS.java
index 6797db4..a5d1a03 100644
--- a/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFS.java
+++ b/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFS.java
@@ -18,16 +18,17 @@
package org.apache.storm.hdfs.security;
-import org.apache.storm.common.AbstractHadoopAutoCreds;
+import static org.apache.storm.hdfs.security.HdfsSecurityUtil.HDFS_CREDENTIALS;
import java.util.Map;
-import static org.apache.storm.hdfs.security.HdfsSecurityUtil.HDFS_CREDENTIALS;
+import org.apache.storm.common.AbstractHadoopAutoCreds;
/**
* Auto credentials plugin for HDFS implementation. This class provides a way to automatically
* push credentials to a topology and to retrieve them in the worker.
*/
+@SuppressWarnings("checkstyle:AbbreviationAsWordInName")
public class AutoHDFS extends AbstractHadoopAutoCreds {
@Override
public void doPrepare(Map<String, Object> conf) {
diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFSCommand.java b/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFSCommand.java
index 4a6a9a4..1e9290a 100644
--- a/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFSCommand.java
+++ b/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFSCommand.java
@@ -15,49 +15,51 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.hdfs.security;
-import org.apache.storm.Config;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import static org.apache.storm.hdfs.security.HdfsSecurityUtil.STORM_KEYTAB_FILE_KEY;
+import static org.apache.storm.hdfs.security.HdfsSecurityUtil.STORM_USER_NAME_KEY;
-import javax.security.auth.Subject;
import java.util.HashMap;
import java.util.Map;
+import javax.security.auth.Subject;
-import static org.apache.storm.hdfs.security.HdfsSecurityUtil.STORM_KEYTAB_FILE_KEY;
-import static org.apache.storm.hdfs.security.HdfsSecurityUtil.STORM_USER_NAME_KEY;
+import org.apache.storm.Config;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
- * Command tool of HDFS credential renewer
+ * Command tool of HDFS credential renewer.
*/
+@SuppressWarnings("checkstyle:AbbreviationAsWordInName")
public final class AutoHDFSCommand {
- private static final Logger LOG = LoggerFactory.getLogger(AutoHDFSCommand.class);
+ private static final Logger LOG = LoggerFactory.getLogger(AutoHDFSCommand.class);
- private AutoHDFSCommand() {
- }
+ private AutoHDFSCommand() {
+ }
- @SuppressWarnings("unchecked")
- public static void main(String[] args) throws Exception {
- Map<String, Object> conf = new HashMap<>();
- conf.put(STORM_USER_NAME_KEY, args[1]); //with realm e.g. hdfs@WITZEND.COM
- conf.put(STORM_KEYTAB_FILE_KEY, args[2]);// /etc/security/keytabs/storm.keytab
+ @SuppressWarnings("unchecked")
+ public static void main(String[] args) throws Exception {
+ Map<String, Object> conf = new HashMap<>();
+ conf.put(STORM_USER_NAME_KEY, args[1]); //with realm e.g. hdfs@WITZEND.COM
+ conf.put(STORM_KEYTAB_FILE_KEY, args[2]);// /etc/security/keytabs/storm.keytab
- AutoHDFS autoHDFS = new AutoHDFS();
- autoHDFS.prepare(conf);
- AutoHDFSNimbus autoHDFSNimbus = new AutoHDFSNimbus();
- autoHDFSNimbus.prepare(conf);
+ AutoHDFS autoHdfs = new AutoHDFS();
+ autoHdfs.prepare(conf);
+ AutoHDFSNimbus autoHdfsNimbus = new AutoHDFSNimbus();
+ autoHdfsNimbus.prepare(conf);
- Map<String,String> creds = new HashMap<>();
- autoHDFSNimbus.populateCredentials(creds, conf, args[0]);
- LOG.info("Got HDFS credentials", autoHDFS.getCredentials(creds));
+ Map<String,String> creds = new HashMap<>();
+ autoHdfsNimbus.populateCredentials(creds, conf, args[0]);
+ LOG.info("Got HDFS credentials", autoHdfs.getCredentials(creds));
- Subject s = new Subject();
- autoHDFS.populateSubject(s, creds);
- LOG.info("Got a Subject "+ s);
+ Subject s = new Subject();
+ autoHdfs.populateSubject(s, creds);
+ LOG.info("Got a Subject " + s);
- autoHDFSNimbus.renew(creds, conf, args[0]);
- LOG.info("renewed credentials", autoHDFS.getCredentials(creds));
- }
+ autoHdfsNimbus.renew(creds, conf, args[0]);
+ LOG.info("renewed credentials", autoHdfs.getCredentials(creds));
+ }
}
diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFSNimbus.java b/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFSNimbus.java
index aab16a9..87ab429 100644
--- a/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFSNimbus.java
+++ b/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFSNimbus.java
@@ -18,6 +18,20 @@
package org.apache.storm.hdfs.security;
+import static org.apache.storm.hdfs.security.HdfsSecurityUtil.HDFS_CREDENTIALS;
+import static org.apache.storm.hdfs.security.HdfsSecurityUtil.STORM_KEYTAB_FILE_KEY;
+import static org.apache.storm.hdfs.security.HdfsSecurityUtil.STORM_USER_NAME_KEY;
+import static org.apache.storm.hdfs.security.HdfsSecurityUtil.TOPOLOGY_HDFS_URI;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectOutputStream;
+import java.net.URI;
+import java.security.PrivilegedAction;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
import org.apache.commons.math3.util.Pair;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -31,24 +45,11 @@ import org.apache.storm.common.AbstractHadoopNimbusPluginAutoCreds;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.ObjectOutputStream;
-import java.net.URI;
-import java.security.PrivilegedAction;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.storm.hdfs.security.HdfsSecurityUtil.HDFS_CREDENTIALS;
-import static org.apache.storm.hdfs.security.HdfsSecurityUtil.STORM_KEYTAB_FILE_KEY;
-import static org.apache.storm.hdfs.security.HdfsSecurityUtil.STORM_USER_NAME_KEY;
-import static org.apache.storm.hdfs.security.HdfsSecurityUtil.TOPOLOGY_HDFS_URI;
-
/**
* Auto credentials nimbus plugin for HDFS implementation. This class automatically
* gets HDFS delegation tokens and push it to user's topology.
*/
+@SuppressWarnings("checkstyle:AbbreviationAsWordInName")
public class AutoHDFSNimbus extends AbstractHadoopNimbusPluginAutoCreds {
private static final Logger LOG = LoggerFactory.getLogger(AutoHDFSNimbus.class);
@@ -57,7 +58,7 @@ public class AutoHDFSNimbus extends AbstractHadoopNimbusPluginAutoCreds {
@Override
public void doPrepare(Map<String, Object> conf) {
- if(conf.containsKey(STORM_KEYTAB_FILE_KEY) && conf.containsKey(STORM_USER_NAME_KEY)) {
+ if (conf.containsKey(STORM_KEYTAB_FILE_KEY) && conf.containsKey(STORM_USER_NAME_KEY)) {
hdfsKeyTab = (String) conf.get(STORM_KEYTAB_FILE_KEY);
hdfsPrincipal = (String) conf.get(STORM_USER_NAME_KEY);
}
@@ -84,19 +85,14 @@ public class AutoHDFSNimbus extends AbstractHadoopNimbusPluginAutoCreds {
return getHadoopCredentials(conf, new Configuration(), topologyOwnerPrincipal);
}
- private Configuration getHadoopConfiguration(Map<String, Object> topoConf, String configKey) {
- Configuration configuration = new Configuration();
- fillHadoopConfiguration(topoConf, configKey, configuration);
- return configuration;
- }
-
@SuppressWarnings("unchecked")
private byte[] getHadoopCredentials(Map<String, Object> conf, final Configuration configuration, final String topologySubmitterUser) {
try {
- if(UserGroupInformation.isSecurityEnabled()) {
+ if (UserGroupInformation.isSecurityEnabled()) {
login(configuration);
- final URI nameNodeURI = conf.containsKey(TOPOLOGY_HDFS_URI) ? new URI(conf.get(TOPOLOGY_HDFS_URI).toString())
+ final URI nameNodeUri = conf.containsKey(TOPOLOGY_HDFS_URI)
+ ? new URI(conf.get(TOPOLOGY_HDFS_URI).toString())
: FileSystem.getDefaultUri(configuration);
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
@@ -107,8 +103,8 @@ public class AutoHDFSNimbus extends AbstractHadoopNimbusPluginAutoCreds {
@Override
public Object run() {
try {
- FileSystem fileSystem = FileSystem.get(nameNodeURI, configuration);
- Credentials credential= proxyUser.getCredentials();
+ FileSystem fileSystem = FileSystem.get(nameNodeUri, configuration);
+ Credentials credential = proxyUser.getCredentials();
if (configuration.get(STORM_USER_NAME_KEY) == null) {
configuration.set(STORM_USER_NAME_KEY, hdfsPrincipal);
@@ -140,6 +136,12 @@ public class AutoHDFSNimbus extends AbstractHadoopNimbusPluginAutoCreds {
}
}
+ private Configuration getHadoopConfiguration(Map<String, Object> topoConf, String configKey) {
+ Configuration configuration = new Configuration();
+ fillHadoopConfiguration(topoConf, configKey, configuration);
+ return configuration;
+ }
+
/**
* {@inheritDoc}
*/
@@ -163,8 +165,9 @@ public class AutoHDFSNimbus extends AbstractHadoopNimbusPluginAutoCreds {
LOG.debug("No tokens found for credentials, skipping renewal.");
}
} catch (Exception e) {
- LOG.warn("could not renew the credentials, one of the possible reason is tokens are beyond " +
- "renewal period so attempting to get new tokens.", e);
+ LOG.warn("could not renew the credentials, one of the possible reason is tokens are beyond "
+ + "renewal period so attempting to get new tokens.",
+ e);
populateCredentials(credentials, topologyConf, topologyOwnerPrincipal);
}
}
diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/HdfsSecurityUtil.java b/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/HdfsSecurityUtil.java
index c5a3f0f..a0c8236 100644
--- a/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/HdfsSecurityUtil.java
+++ b/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/HdfsSecurityUtil.java
@@ -15,22 +15,24 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.storm.hdfs.security;
-import org.apache.storm.security.auth.kerberos.AutoTGT;
+package org.apache.storm.hdfs.security;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import static org.apache.storm.Config.TOPOLOGY_AUTO_CREDENTIALS;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
-import static org.apache.storm.Config.TOPOLOGY_AUTO_CREDENTIALS;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+
+import org.apache.storm.security.auth.kerberos.AutoTGT;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* This class provides util methods for storm-hdfs connector communicating
@@ -51,9 +53,9 @@ public final class HdfsSecurityUtil {
public static void login(Map<String, Object> conf, Configuration hdfsConfig) throws IOException {
//If AutoHDFS is specified, do not attempt to login using keytabs, only kept for backward compatibility.
- if(conf.get(TOPOLOGY_AUTO_CREDENTIALS) == null ||
- (!(((List)conf.get(TOPOLOGY_AUTO_CREDENTIALS)).contains(AutoHDFS.class.getName())) &&
- !(((List)conf.get(TOPOLOGY_AUTO_CREDENTIALS)).contains(AutoTGT.class.getName())))) {
+ if (conf.get(TOPOLOGY_AUTO_CREDENTIALS) == null
+ || (!(((List)conf.get(TOPOLOGY_AUTO_CREDENTIALS)).contains(AutoHDFS.class.getName()))
+ && !(((List)conf.get(TOPOLOGY_AUTO_CREDENTIALS)).contains(AutoTGT.class.getName())))) {
if (UserGroupInformation.isSecurityEnabled()) {
// compareAndSet added because of https://issues.apache.org/jira/browse/STORM-1535
if (isLoggedIn.compareAndSet(false, true)) {
diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hive/security/AutoHive.java b/external/storm-autocreds/src/main/java/org/apache/storm/hive/security/AutoHive.java
index 2160753..94b38fa 100644
--- a/external/storm-autocreds/src/main/java/org/apache/storm/hive/security/AutoHive.java
+++ b/external/storm-autocreds/src/main/java/org/apache/storm/hive/security/AutoHive.java
@@ -18,12 +18,12 @@
package org.apache.storm.hive.security;
-import org.apache.storm.common.AbstractHadoopAutoCreds;
+import static org.apache.storm.hive.security.HiveSecurityUtil.HIVE_CREDENTIALS;
+import static org.apache.storm.hive.security.HiveSecurityUtil.HIVE_CREDENTIALS_CONFIG_KEYS;
import java.util.Map;
-import static org.apache.storm.hive.security.HiveSecurityUtil.HIVE_CREDENTIALS;
-import static org.apache.storm.hive.security.HiveSecurityUtil.HIVE_CREDENTIALS_CONFIG_KEYS;
+import org.apache.storm.common.AbstractHadoopAutoCreds;
/**
* Auto credentials plugin for Hive implementation. This class provides a way to automatically
diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hive/security/AutoHiveCommand.java b/external/storm-autocreds/src/main/java/org/apache/storm/hive/security/AutoHiveCommand.java
index dcd99ae..f5c139b 100644
--- a/external/storm-autocreds/src/main/java/org/apache/storm/hive/security/AutoHiveCommand.java
+++ b/external/storm-autocreds/src/main/java/org/apache/storm/hive/security/AutoHiveCommand.java
@@ -15,51 +15,53 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.hive.security;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.storm.Config;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import static org.apache.storm.hive.security.HiveSecurityUtil.HIVE_KEYTAB_FILE_KEY;
+import static org.apache.storm.hive.security.HiveSecurityUtil.HIVE_PRINCIPAL_KEY;
-import javax.security.auth.Subject;
import java.util.HashMap;
import java.util.Map;
+import javax.security.auth.Subject;
-import static org.apache.storm.hive.security.HiveSecurityUtil.HIVE_KEYTAB_FILE_KEY;
-import static org.apache.storm.hive.security.HiveSecurityUtil.HIVE_PRINCIPAL_KEY;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.storm.Config;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
- * Command tool of Hive credential renewer
+ * Command tool of Hive credential renewer.
*/
public final class AutoHiveCommand {
- private static final Logger LOG = LoggerFactory.getLogger(AutoHiveCommand.class);
+ private static final Logger LOG = LoggerFactory.getLogger(AutoHiveCommand.class);
- private AutoHiveCommand() {
- }
+ private AutoHiveCommand() {
+ }
- @SuppressWarnings("unchecked")
- public static void main(String[] args) throws Exception {
- Map<String, Object> conf = new HashMap<>();
- conf.put(HIVE_PRINCIPAL_KEY, args[1]); // hive principal storm-hive@WITZEN.COM
- conf.put(HIVE_KEYTAB_FILE_KEY, args[2]); // storm hive keytab /etc/security/keytabs/storm-hive.keytab
- conf.put(HiveConf.ConfVars.METASTOREURIS.varname, args[3]); // hive.metastore.uris : "thrift://pm-eng1-cluster1.field.hortonworks.com:9083"
+ @SuppressWarnings("unchecked")
+ public static void main(String[] args) throws Exception {
+ Map<String, Object> conf = new HashMap<>();
+ conf.put(HIVE_PRINCIPAL_KEY, args[1]); // hive principal storm-hive@WITZEN.COM
+ conf.put(HIVE_KEYTAB_FILE_KEY, args[2]); // storm hive keytab /etc/security/keytabs/storm-hive.keytab
+ // hive.metastore.uris : "thrift://pm-eng1-cluster1.field.hortonworks.com:9083"
+ conf.put(HiveConf.ConfVars.METASTOREURIS.varname, args[3]);
- AutoHive autoHive = new AutoHive();
- autoHive.prepare(conf);
- AutoHiveNimbus autoHiveNimbus = new AutoHiveNimbus();
- autoHiveNimbus.prepare(conf);
+ AutoHive autoHive = new AutoHive();
+ autoHive.prepare(conf);
+ AutoHiveNimbus autoHiveNimbus = new AutoHiveNimbus();
+ autoHiveNimbus.prepare(conf);
- Map<String, String> creds = new HashMap<>();
- autoHiveNimbus.populateCredentials(creds, conf, args[0]);
- LOG.info("Got Hive credentials" + autoHive.getCredentials(creds));
+ Map<String, String> creds = new HashMap<>();
+ autoHiveNimbus.populateCredentials(creds, conf, args[0]);
+ LOG.info("Got Hive credentials" + autoHive.getCredentials(creds));
- Subject subject = new Subject();
- autoHive.populateSubject(subject, creds);
- LOG.info("Got a Subject " + subject);
+ Subject subject = new Subject();
+ autoHive.populateSubject(subject, creds);
+ LOG.info("Got a Subject " + subject);
- autoHiveNimbus.renew(creds, conf, args[0]);
- LOG.info("Renewed credentials" + autoHive.getCredentials(creds));
- }
+ autoHiveNimbus.renew(creds, conf, args[0]);
+ LOG.info("Renewed credentials" + autoHive.getCredentials(creds));
+ }
}
diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hive/security/AutoHiveNimbus.java b/external/storm-autocreds/src/main/java/org/apache/storm/hive/security/AutoHiveNimbus.java
index 9854b69..e3b0ca7 100644
--- a/external/storm-autocreds/src/main/java/org/apache/storm/hive/security/AutoHiveNimbus.java
+++ b/external/storm-autocreds/src/main/java/org/apache/storm/hive/security/AutoHiveNimbus.java
@@ -18,6 +18,20 @@
package org.apache.storm.hive.security;
+import static org.apache.storm.hive.security.HiveSecurityUtil.HIVE_CREDENTIALS;
+import static org.apache.storm.hive.security.HiveSecurityUtil.HIVE_CREDENTIALS_CONFIG_KEYS;
+import static org.apache.storm.hive.security.HiveSecurityUtil.HIVE_KEYTAB_FILE_KEY;
+import static org.apache.storm.hive.security.HiveSecurityUtil.HIVE_PRINCIPAL_KEY;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.ObjectOutputStream;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
import org.apache.commons.math3.util.Pair;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -29,25 +43,10 @@ import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hive.hcatalog.api.HCatClient;
import org.apache.hive.hcatalog.common.HCatException;
-import org.apache.storm.Config;
import org.apache.storm.common.AbstractHadoopNimbusPluginAutoCreds;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.IOException;
-import java.io.ObjectOutputStream;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.storm.hive.security.HiveSecurityUtil.HIVE_CREDENTIALS;
-import static org.apache.storm.hive.security.HiveSecurityUtil.HIVE_CREDENTIALS_CONFIG_KEYS;
-import static org.apache.storm.hive.security.HiveSecurityUtil.HIVE_KEYTAB_FILE_KEY;
-import static org.apache.storm.hive.security.HiveSecurityUtil.HIVE_PRINCIPAL_KEY;
-
/**
* Auto credentials nimbus plugin for Hive implementation. This class automatically
* gets Hive delegation tokens and push it to user's topology.
@@ -57,6 +56,7 @@ public class AutoHiveNimbus extends AbstractHadoopNimbusPluginAutoCreds {
public String hiveKeytab;
public String hivePrincipal;
+ @SuppressWarnings("checkstyle:AbbreviationAsWordInName")
public String metaStoreURI;
@Override
@@ -90,29 +90,13 @@ public class AutoHiveNimbus extends AbstractHadoopNimbusPluginAutoCreds {
return getHadoopCredentials(conf, configuration, topologyOwnerPrincipal);
}
- private Configuration getHadoopConfiguration(Map<String, Object> topoConf, String configKey) {
- Configuration configuration = new Configuration();
- fillHadoopConfiguration(topoConf, configKey, configuration);
- return configuration;
- }
-
- public HiveConf createHiveConf(String metaStoreURI, String hiveMetaStorePrincipal) throws IOException {
- HiveConf hcatConf = new HiveConf();
- hcatConf.setVar(HiveConf.ConfVars.METASTOREURIS, metaStoreURI);
- hcatConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3);
- hcatConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
- hcatConf.setBoolVar(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL, true);
- hcatConf.set(HiveConf.ConfVars.METASTORE_KERBEROS_PRINCIPAL.varname, hiveMetaStorePrincipal);
- return hcatConf;
- }
-
@SuppressWarnings("unchecked")
protected byte[] getHadoopCredentials(Map<String, Object> conf, final Configuration configuration, final String topologySubmitterUser) {
try {
if (UserGroupInformation.isSecurityEnabled()) {
- String hiveMetaStoreURI = getMetaStoreURI(configuration);
+ String hiveMetaStoreUri = getMetaStoreUri(configuration);
String hiveMetaStorePrincipal = getMetaStorePrincipal(configuration);
- HiveConf hcatConf = createHiveConf(hiveMetaStoreURI, hiveMetaStorePrincipal);
+ HiveConf hcatConf = createHiveConf(hiveMetaStoreUri, hiveMetaStorePrincipal);
login(configuration);
UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
@@ -142,6 +126,22 @@ public class AutoHiveNimbus extends AbstractHadoopNimbusPluginAutoCreds {
}
}
+ private Configuration getHadoopConfiguration(Map<String, Object> topoConf, String configKey) {
+ Configuration configuration = new Configuration();
+ fillHadoopConfiguration(topoConf, configKey, configuration);
+ return configuration;
+ }
+
+ public HiveConf createHiveConf(String metaStoreUri, String hiveMetaStorePrincipal) throws IOException {
+ HiveConf hcatConf = new HiveConf();
+ hcatConf.setVar(HiveConf.ConfVars.METASTOREURIS, metaStoreUri);
+ hcatConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3);
+ hcatConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
+ hcatConf.setBoolVar(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL, true);
+ hcatConf.set(HiveConf.ConfVars.METASTORE_KERBEROS_PRINCIPAL.varname, hiveMetaStorePrincipal);
+ return hcatConf;
+ }
+
private Token<DelegationTokenIdentifier> getDelegationToken(HiveConf hcatConf,
String metaStoreServicePrincipal,
String topologySubmitterUser) throws IOException {
@@ -161,23 +161,26 @@ public class AutoHiveNimbus extends AbstractHadoopNimbusPluginAutoCreds {
return delegationTokenId;
} finally {
- if (hcatClient != null)
+ if (hcatClient != null) {
hcatClient.close();
+ }
}
}
- private String getMetaStoreURI(Configuration configuration) {
- if (configuration.get(HiveConf.ConfVars.METASTOREURIS.varname) == null)
+ private String getMetaStoreUri(Configuration configuration) {
+ if (configuration.get(HiveConf.ConfVars.METASTOREURIS.varname) == null) {
return metaStoreURI;
- else
+ } else {
return configuration.get(HiveConf.ConfVars.METASTOREURIS.varname);
+ }
}
private String getMetaStorePrincipal(Configuration configuration) {
- if (configuration.get(HIVE_PRINCIPAL_KEY) == null)
+ if (configuration.get(HIVE_PRINCIPAL_KEY) == null) {
return hivePrincipal;
- else
+ } else {
return configuration.get(HIVE_PRINCIPAL_KEY);
+ }
}
private void login(Configuration configuration) throws IOException {
@@ -197,7 +200,7 @@ public class AutoHiveNimbus extends AbstractHadoopNimbusPluginAutoCreds {
for (Pair<String, Credentials> cred : getCredentials(credentials, configKeys)) {
try {
Configuration configuration = getHadoopConfiguration(topologyConf, cred.getFirst());
- String hiveMetaStoreURI = getMetaStoreURI(configuration);
+ String hiveMetaStoreUri = getMetaStoreUri(configuration);
String hiveMetaStorePrincipal = getMetaStorePrincipal(configuration);
Collection<Token<? extends TokenIdentifier>> tokens = cred.getSecond().getAllTokens();
@@ -205,26 +208,27 @@ public class AutoHiveNimbus extends AbstractHadoopNimbusPluginAutoCreds {
if (tokens != null && !tokens.isEmpty()) {
for (Token<? extends TokenIdentifier> token : tokens) {
- long expiration = renewToken(token, hiveMetaStoreURI, hiveMetaStorePrincipal);
+ long expiration = renewToken(token, hiveMetaStoreUri, hiveMetaStorePrincipal);
LOG.info("Hive delegation token renewed, new expiration time {}", expiration);
}
} else {
LOG.debug("No tokens found for credentials, skipping renewal.");
}
} catch (Exception e) {
- LOG.warn("could not renew the credentials, one of the possible reason is tokens are beyond " +
- "renewal period so attempting to get new tokens.", e);
+ LOG.warn("could not renew the credentials, one of the possible reason is tokens are beyond "
+ + "renewal period so attempting to get new tokens.",
+ e);
populateCredentials(credentials, topologyConf);
}
}
}
- private long renewToken(Token token, String metaStoreURI, String hiveMetaStorePrincipal) {
+ private long renewToken(Token token, String metaStoreUri, String hiveMetaStorePrincipal) {
HCatClient hcatClient = null;
if (UserGroupInformation.isSecurityEnabled()) {
try {
String tokenStr = token.encodeToUrlString();
- HiveConf hcatConf = createHiveConf(metaStoreURI, hiveMetaStorePrincipal);
+ HiveConf hcatConf = createHiveConf(metaStoreUri, hiveMetaStorePrincipal);
LOG.debug("renewing delegation tokens for principal={}", hiveMetaStorePrincipal);
hcatClient = HCatClient.create(hcatConf);
Long expiryTime = hcatClient.renewDelegationToken(tokenStr);
@@ -233,12 +237,13 @@ public class AutoHiveNimbus extends AbstractHadoopNimbusPluginAutoCreds {
} catch (Exception ex) {
throw new RuntimeException("Failed to renew delegation tokens.", ex);
} finally {
- if (hcatClient != null)
+ if (hcatClient != null) {
try {
hcatClient.close();
} catch (HCatException e) {
LOG.error(" Exception", e);
}
+ }
}
} else {
throw new RuntimeException("Security is not enabled for Hadoop");
diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hive/security/HiveSecurityUtil.java b/external/storm-autocreds/src/main/java/org/apache/storm/hive/security/HiveSecurityUtil.java
index 3e3186f..de67f2c 100644
--- a/external/storm-autocreds/src/main/java/org/apache/storm/hive/security/HiveSecurityUtil.java
+++ b/external/storm-autocreds/src/main/java/org/apache/storm/hive/security/HiveSecurityUtil.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.hive.security;
/**
@@ -22,11 +23,11 @@ package org.apache.storm.hive.security;
* with secured Hive.
*/
public final class HiveSecurityUtil {
- public static final String HIVE_KEYTAB_FILE_KEY = "hive.keytab.file";
- public static final String HIVE_PRINCIPAL_KEY = "hive.kerberos.principal";
- public static final String HIVE_CREDENTIALS_CONFIG_KEYS = "hiveCredentialsConfigKeys";
- public static final String HIVE_CREDENTIALS = "HIVE_CREDENTIALS";
+ public static final String HIVE_KEYTAB_FILE_KEY = "hive.keytab.file";
+ public static final String HIVE_PRINCIPAL_KEY = "hive.kerberos.principal";
+ public static final String HIVE_CREDENTIALS_CONFIG_KEYS = "hiveCredentialsConfigKeys";
+ public static final String HIVE_CREDENTIALS = "HIVE_CREDENTIALS";
- private HiveSecurityUtil() {
- }
+ private HiveSecurityUtil() {
+ }
}