You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by mc...@apache.org on 2016/04/04 16:14:40 UTC
nifi git commit: NIFI-1698 Improving customValidate in
AbstractHadoopProcessor and HBaseClient service to not reload Configuration
unless it changed. This closes #313
Repository: nifi
Updated Branches:
refs/heads/master e02c79975 -> 04c683093
NIFI-1698 Improving customValidate in AbstractHadoopProcessor and HBaseClient service to not reload Configuration unless it changed. This closes #313
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/04c68309
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/04c68309
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/04c68309
Branch: refs/heads/master
Commit: 04c683093746da26ec72da05a530d61471d8a6d4
Parents: e02c799
Author: Bryan Bende <bb...@apache.org>
Authored: Wed Mar 30 15:56:22 2016 -0400
Committer: Matt Gilman <ma...@gmail.com>
Committed: Mon Apr 4 10:14:05 2016 -0400
----------------------------------------------------------------------
.../org/apache/nifi/hadoop/SecurityUtil.java | 2 +-
.../hadoop/AbstractHadoopProcessor.java | 35 ++++++++++++++++++--
.../nifi/hbase/HBase_1_1_2_ClientService.java | 35 +++++++++++++++++++-
3 files changed, 68 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/04c68309/nifi-commons/nifi-hadoop-utils/src/main/java/org/apache/nifi/hadoop/SecurityUtil.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-hadoop-utils/src/main/java/org/apache/nifi/hadoop/SecurityUtil.java b/nifi-commons/nifi-hadoop-utils/src/main/java/org/apache/nifi/hadoop/SecurityUtil.java
index 74197ef..8d8c6e9 100644
--- a/nifi-commons/nifi-hadoop-utils/src/main/java/org/apache/nifi/hadoop/SecurityUtil.java
+++ b/nifi-commons/nifi-hadoop-utils/src/main/java/org/apache/nifi/hadoop/SecurityUtil.java
@@ -79,7 +79,7 @@ public class SecurityUtil {
* @return true if kerberos is enabled on the given configuration, false otherwise
*
*/
- public static synchronized boolean isSecurityEnabled(final Configuration config) {
+ public static boolean isSecurityEnabled(final Configuration config) {
Validate.notNull(config);
return "kerberos".equalsIgnoreCase(config.get("hadoop.security.authentication"));
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/04c68309/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/AbstractHadoopProcessor.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/AbstractHadoopProcessor.java b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/AbstractHadoopProcessor.java
index cd9683b..9efc0f6 100644
--- a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/AbstractHadoopProcessor.java
+++ b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/AbstractHadoopProcessor.java
@@ -116,6 +116,9 @@ public abstract class AbstractHadoopProcessor extends AbstractProcessor {
// Hadoop Configuration, Filesystem, and UserGroupInformation (optional)
private final AtomicReference<HdfsResources> hdfsResources = new AtomicReference<>();
+ // Holder of cached Configuration information so validation does not reload the same config over and over
+ private final AtomicReference<ValidationResources> validationResourceHolder = new AtomicReference<>();
+
@Override
protected void init(ProcessorInitializationContext context) {
hdfsResources.set(new HdfsResources(null, null, null));
@@ -147,12 +150,21 @@ public abstract class AbstractHadoopProcessor extends AbstractProcessor {
final List<ValidationResult> results = new ArrayList<>();
if (!StringUtils.isBlank(configResources)) {
- Configuration conf = null;
try {
- conf = getConfigurationFromResources(configResources);
+ ValidationResources resources = validationResourceHolder.get();
+
+ // if no resources in the holder, or if the holder has different resources loaded,
+ // then load the Configuration and set the new resources in the holder
+ if (resources == null || !configResources.equals(resources.getConfigResources())) {
+ getLogger().debug("Reloading validation resources");
+ resources = new ValidationResources(configResources, getConfigurationFromResources(configResources));
+ validationResourceHolder.set(resources);
+ }
+ final Configuration conf = resources.getConfiguration();
results.addAll(KerberosProperties.validatePrincipalAndKeytab(
this.getClass().getSimpleName(), conf, principal, keytab, getLogger()));
+
} catch (IOException e) {
results.add(new ValidationResult.Builder()
.valid(false)
@@ -452,4 +464,23 @@ public abstract class AbstractHadoopProcessor extends AbstractProcessor {
return userGroupInformation;
}
}
+
+ static protected class ValidationResources {
+ private final String configResources;
+ private final Configuration configuration;
+
+ public ValidationResources(String configResources, Configuration configuration) {
+ this.configResources = configResources;
+ this.configuration = configuration;
+ }
+
+ public String getConfigResources() {
+ return configResources;
+ }
+
+ public Configuration getConfiguration() {
+ return configuration;
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/04c68309/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/main/java/org/apache/nifi/hbase/HBase_1_1_2_ClientService.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/main/java/org/apache/nifi/hbase/HBase_1_1_2_ClientService.java b/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/main/java/org/apache/nifi/hbase/HBase_1_1_2_ClientService.java
index 3465135..1791cfe 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/main/java/org/apache/nifi/hbase/HBase_1_1_2_ClientService.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-hbase_1_1_2-client-service-bundle/nifi-hbase_1_1_2-client-service/src/main/java/org/apache/nifi/hbase/HBase_1_1_2_ClientService.java
@@ -65,6 +65,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
@Tags({ "hbase", "client"})
@CapabilityDescription("Implementation of HBaseClientService for HBase 1.1.2. This service can be configured by providing " +
@@ -90,6 +91,9 @@ public class HBase_1_1_2_ClientService extends AbstractControllerService impleme
private List<PropertyDescriptor> properties;
private KerberosProperties kerberosProperties;
+ // Holder of cached Configuration information so validation does not reload the same config over and over
+ private final AtomicReference<ValidationResources> validationResourceHolder = new AtomicReference<>();
+
@Override
protected void init(ControllerServiceInitializationContext config) throws InitializationException {
this.kerberosProperties = getKerberosProperties();
@@ -145,7 +149,17 @@ public class HBase_1_1_2_ClientService extends AbstractControllerService impleme
if (confFileProvided) {
final String configFiles = validationContext.getProperty(HADOOP_CONF_FILES).getValue();
- final Configuration hbaseConfig = getConfigurationFromFiles(configFiles);
+ ValidationResources resources = validationResourceHolder.get();
+
+ // if no resources in the holder, or if the holder has different resources loaded,
+ // then load the Configuration and set the new resources in the holder
+ if (resources == null || !configFiles.equals(resources.getConfigResources())) {
+ getLogger().debug("Reloading validation resources");
+ resources = new ValidationResources(configFiles, getConfigurationFromFiles(configFiles));
+ validationResourceHolder.set(resources);
+ }
+
+ final Configuration hbaseConfig = resources.getConfiguration();
final String principal = validationContext.getProperty(kerberosProperties.getKerberosPrincipal()).getValue();
final String keytab = validationContext.getProperty(kerberosProperties.getKerberosKeytab()).getValue();
@@ -372,4 +386,23 @@ public class HBase_1_1_2_ClientService extends AbstractControllerService impleme
return table.getScanner(scan);
}
+
+ static protected class ValidationResources {
+ private final String configResources;
+ private final Configuration configuration;
+
+ public ValidationResources(String configResources, Configuration configuration) {
+ this.configResources = configResources;
+ this.configuration = configuration;
+ }
+
+ public String getConfigResources() {
+ return configResources;
+ }
+
+ public Configuration getConfiguration() {
+ return configuration;
+ }
+ }
+
}