You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@knox.apache.org by sm...@apache.org on 2023/12/12 13:49:14 UTC
(knox) branch master updated: KNOX-2956 - Removing CM-specific 'advanced service discovery' handler and have everything process by the HXR parser (#821)
This is an automated email from the ASF dual-hosted git repository.
smolnar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/knox.git
The following commit(s) were added to refs/heads/master by this push:
new 14954a0f1 KNOX-2956 - Removing CM-specific 'advanced service discovery' handler and have everything process by the HXR parser (#821)
14954a0f1 is described below
commit 14954a0f1614ab6c4d4120bf701b8f6f5f414a40
Author: Sandor Molnar <sm...@apache.org>
AuthorDate: Tue Dec 12 14:49:08 2023 +0100
KNOX-2956 - Removing CM-specific 'advanced service discovery' handler and have everything process by the HXR parser (#821)
Change-Id: Ib1837610e4b222282af7bef98fc6f27af5169e88
---
.../org/apache/knox/gateway/GatewayServer.java | 5 -
...vanceServiceDiscoveryConfigurationMessages.java | 39 -------
.../advanced/AdvancedServiceDiscoveryConfig.java | 88 ----------------
...vancedServiceDiscoveryConfigChangeListener.java | 28 -----
...vancedServiceDiscoveryConfigurationMonitor.java | 116 ---------------------
.../hadoop/xml/HadoopXmlResourceMessages.java | 11 +-
.../hadoop/xml/HadoopXmlResourceMonitor.java | 32 ++----
.../hadoop/xml/HadoopXmlResourceParser.java | 111 ++++++--------------
.../hadoop/xml/HadoopXmlResourceParserTest.java | 80 +++-----------
.../resources/testDescriptorWithServiceList.xml | 34 ++++++
10 files changed, 96 insertions(+), 448 deletions(-)
diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/GatewayServer.java b/gateway-server/src/main/java/org/apache/knox/gateway/GatewayServer.java
index ce62b1c79..01c04cf7f 100644
--- a/gateway-server/src/main/java/org/apache/knox/gateway/GatewayServer.java
+++ b/gateway-server/src/main/java/org/apache/knox/gateway/GatewayServer.java
@@ -48,7 +48,6 @@ import org.apache.knox.gateway.topology.Application;
import org.apache.knox.gateway.topology.Topology;
import org.apache.knox.gateway.topology.TopologyEvent;
import org.apache.knox.gateway.topology.TopologyListener;
-import org.apache.knox.gateway.topology.discovery.advanced.AdvancedServiceDiscoveryConfigurationMonitor;
import org.apache.knox.gateway.topology.hadoop.xml.HadoopXmlResourceMonitor;
import org.apache.knox.gateway.topology.hadoop.xml.HadoopXmlResourceParser;
import org.apache.knox.gateway.trace.AccessHandler;
@@ -719,10 +718,6 @@ public class GatewayServer {
private void handleHadoopXmlResources() {
final HadoopXmlResourceParser hadoopXmlResourceParser = new HadoopXmlResourceParser(config);
final HadoopXmlResourceMonitor hadoopXmlResourceMonitor = new HadoopXmlResourceMonitor(config, hadoopXmlResourceParser);
- final AdvancedServiceDiscoveryConfigurationMonitor advancedServiceDiscoveryConfigurationMonitor = new AdvancedServiceDiscoveryConfigurationMonitor(config);
- advancedServiceDiscoveryConfigurationMonitor.registerListener(hadoopXmlResourceParser);
- advancedServiceDiscoveryConfigurationMonitor.registerListener(hadoopXmlResourceMonitor);
- advancedServiceDiscoveryConfigurationMonitor.init();
hadoopXmlResourceMonitor.setupMonitor();
}
diff --git a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/discovery/advanced/AdvanceServiceDiscoveryConfigurationMessages.java b/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/discovery/advanced/AdvanceServiceDiscoveryConfigurationMessages.java
deleted file mode 100644
index b2a4f6316..000000000
--- a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/discovery/advanced/AdvanceServiceDiscoveryConfigurationMessages.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.knox.gateway.topology.discovery.advanced;
-
-import org.apache.knox.gateway.i18n.messages.Message;
-import org.apache.knox.gateway.i18n.messages.MessageLevel;
-import org.apache.knox.gateway.i18n.messages.Messages;
-import org.apache.knox.gateway.i18n.messages.StackTrace;
-
-@Messages(logger = "org.apache.knox.gateway.topology.discovery.advanced")
-public interface AdvanceServiceDiscoveryConfigurationMessages {
-
- @Message(level = MessageLevel.INFO, text = "Monitoring {0}/{1}* for advanced service discovery configuration changes.")
- void monitorStarted(String directory, String prefix);
-
- @Message(level = MessageLevel.INFO, text = "Monitoring advanced service discovery configuration changes is disabled.")
- void disableMonitoring();
-
- @Message(level = MessageLevel.ERROR, text = "Error while monitoring CM advanced configuration: {0}")
- void failedToMonitorClouderaManagerAdvancedConfiguration(String errorMessage, @StackTrace(level = MessageLevel.DEBUG) Exception e);
-
- @Message(level = MessageLevel.INFO, text = "Notifying listeners due to advanced service discovery configuration changes in {0}...")
- void notifyListeners(String path);
-
-}
diff --git a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/discovery/advanced/AdvancedServiceDiscoveryConfig.java b/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/discovery/advanced/AdvancedServiceDiscoveryConfig.java
deleted file mode 100644
index c463bea68..000000000
--- a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/discovery/advanced/AdvancedServiceDiscoveryConfig.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.knox.gateway.topology.discovery.advanced;
-
-import static java.util.stream.Collectors.toSet;
-
-import java.util.Locale;
-import java.util.Map.Entry;
-import java.util.Properties;
-import java.util.Set;
-
-/**
- * Wrapper class providing useful methods on properties coming from
- * <code>$KNOX_CONF_DIR/auto-discovery-advanced-configuration.properties</code>
- */
-public class AdvancedServiceDiscoveryConfig {
-
- public static final String PARAMETER_NAME_PREFIX_ENABLED_SERVICE = "gateway.auto.discovery.";
- public static final String PARAMETER_NAME_POSTFIX_ENABLED_SERVICE = ".enabled.";
- public static final String PARAMETER_NAME_TOPOLOGY_NAME = "gateway.auto.discovery.topology.name";
- public static final String PARAMETER_NAME_DISCOVERY_ADDRESS = "gateway.auto.discovery.address";
- public static final String PARAMETER_NAME_DISCOVERY_CLUSTER = "gateway.auto.discovery.cluster";
-
- private final Properties properties;
-
- public AdvancedServiceDiscoveryConfig() {
- this(null);
- }
-
- public AdvancedServiceDiscoveryConfig(Properties properties) {
- this.properties = properties == null ? new Properties() : properties;
- }
-
- public boolean isServiceEnabled(String serviceName) {
- final String propertyName = PARAMETER_NAME_PREFIX_ENABLED_SERVICE + getTopologyName() + PARAMETER_NAME_POSTFIX_ENABLED_SERVICE + serviceName;
- return Boolean.valueOf(getPropertyIgnoreCase(propertyName, "true"));
- }
-
- public Set<String> getEnabledServiceNames() {
- return properties.entrySet().stream().filter(keyValuePair -> Boolean.valueOf((String) keyValuePair.getValue()))
- .map(keyValuePair -> ((String) keyValuePair.getKey()).substring(((String) keyValuePair.getKey()).lastIndexOf('.') + 1).toUpperCase(Locale.getDefault())).collect(toSet());
- }
-
- public String getTopologyName() {
- return getPropertyIgnoreCase(PARAMETER_NAME_TOPOLOGY_NAME, "");
- }
-
- public String getDiscoveryAddress() {
- return getPropertyIgnoreCase(PARAMETER_NAME_DISCOVERY_ADDRESS, "");
- }
-
- public String getDiscoveryCluster() {
- return getPropertyIgnoreCase(PARAMETER_NAME_DISCOVERY_CLUSTER, "");
- }
-
- private String getPropertyIgnoreCase(String propertyName, String defaultValue) {
- final String property = properties.getProperty(propertyName);
- if (property != null) {
- return property;
- } else {
- for (Entry<Object, Object> entry : properties.entrySet()) {
- if (propertyName.equalsIgnoreCase((String) entry.getKey())) {
- return (String) entry.getValue();
- }
- }
- return defaultValue;
- }
- }
-
- @Override
- public String toString() {
- return this.properties.toString();
- }
-}
diff --git a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/discovery/advanced/AdvancedServiceDiscoveryConfigChangeListener.java b/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/discovery/advanced/AdvancedServiceDiscoveryConfigChangeListener.java
deleted file mode 100644
index 21061ecd2..000000000
--- a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/discovery/advanced/AdvancedServiceDiscoveryConfigChangeListener.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.knox.gateway.topology.discovery.advanced;
-
-import java.util.Properties;
-
-/**
- * The listener interface for receiving service discovery configuration events.
- */
-public interface AdvancedServiceDiscoveryConfigChangeListener {
-
- void onAdvancedServiceDiscoveryConfigurationChange(Properties newConfiguration);
-
-}
diff --git a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/discovery/advanced/AdvancedServiceDiscoveryConfigurationMonitor.java b/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/discovery/advanced/AdvancedServiceDiscoveryConfigurationMonitor.java
deleted file mode 100644
index c314a4d52..000000000
--- a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/discovery/advanced/AdvancedServiceDiscoveryConfigurationMonitor.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.knox.gateway.topology.discovery.advanced;
-
-import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
-
-import java.io.File;
-import java.io.FileFilter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.attribute.FileTime;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.io.filefilter.PrefixFileFilter;
-import org.apache.commons.lang3.concurrent.BasicThreadFactory;
-import org.apache.knox.gateway.config.GatewayConfig;
-import org.apache.knox.gateway.i18n.messages.MessagesFactory;
-
-/**
- * Monitoring <code>$KNOX_CONF_DIR/auto-discovery-advanced-configuration.properties</code> (if exists) and notifies any
- * {@link AdvancedServiceDiscoveryConfigChangeListener} if the file is changed since the last time it was loaded
- *
- */
-public class AdvancedServiceDiscoveryConfigurationMonitor {
-
- private static final String ADVANCED_CONFIGURATION_FILE_NAME_PREFIX = "auto-discovery-advanced-configuration-";
- private static final AdvanceServiceDiscoveryConfigurationMessages LOG = MessagesFactory.get(AdvanceServiceDiscoveryConfigurationMessages.class);
-
- private final List<AdvancedServiceDiscoveryConfigChangeListener> listeners;
- private final String gatewayConfigurationDir;
- private final long monitoringInterval;
- private final Map<Path, FileTime> lastReloadTimes;
-
- public AdvancedServiceDiscoveryConfigurationMonitor(GatewayConfig gatewayConfig) {
- this.gatewayConfigurationDir = gatewayConfig.getGatewayConfDir();
- this.monitoringInterval = gatewayConfig.getClouderaManagerAdvancedServiceDiscoveryConfigurationMonitoringInterval();
- this.listeners = new ArrayList<>();
- this.lastReloadTimes = new ConcurrentHashMap<>();
- }
-
- public void init() {
- monitorAdvancedServiceConfigurations();
- setupMonitor();
- }
-
- private void setupMonitor() {
- if (monitoringInterval > 0) {
- final ScheduledExecutorService executorService = newSingleThreadScheduledExecutor(new BasicThreadFactory.Builder().namingPattern("AdvancedServiceDiscoveryConfigurationMonitor-%d").build());
- executorService.scheduleAtFixedRate(() -> monitorAdvancedServiceConfigurations(), 0, monitoringInterval, TimeUnit.MILLISECONDS);
- LOG.monitorStarted(gatewayConfigurationDir, ADVANCED_CONFIGURATION_FILE_NAME_PREFIX);
- } else {
- LOG.disableMonitoring();
- }
- }
-
- public void registerListener(AdvancedServiceDiscoveryConfigChangeListener listener) {
- listeners.add(listener);
- }
-
- private void monitorAdvancedServiceConfigurations() {
- final File[] advancedConfigurationFiles = new File(gatewayConfigurationDir).listFiles((FileFilter) new PrefixFileFilter(ADVANCED_CONFIGURATION_FILE_NAME_PREFIX));
- if (advancedConfigurationFiles != null) {
- for (File advancedConfigurationFile : advancedConfigurationFiles) {
- monitorAdvancedServiceConfiguration(Paths.get(advancedConfigurationFile.getAbsolutePath()));
- }
- }
- }
-
- private void monitorAdvancedServiceConfiguration(Path resourcePath) {
- try {
- if (Files.exists(resourcePath) && Files.isReadable(resourcePath)) {
- FileTime lastModifiedTime = Files.getLastModifiedTime(resourcePath);
- FileTime lastReloadTime = lastReloadTimes.get(resourcePath);
- if (lastReloadTime == null || lastReloadTime.compareTo(lastModifiedTime) < 0) {
- lastReloadTimes.put(resourcePath, lastModifiedTime);
- try (InputStream advanceconfigurationFileInputStream = Files.newInputStream(resourcePath)) {
- Properties properties = new Properties();
- properties.load(advanceconfigurationFileInputStream);
- notifyListeners(resourcePath.toString(), properties);
- }
- }
- }
- } catch (IOException e) {
- LOG.failedToMonitorClouderaManagerAdvancedConfiguration(e.getMessage(), e);
- }
- }
-
- private void notifyListeners(String path, Properties properties) {
- LOG.notifyListeners(path);
- listeners.forEach(listener -> listener.onAdvancedServiceDiscoveryConfigurationChange(properties));
- }
-
-}
diff --git a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceMessages.java b/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceMessages.java
index 063fbad4b..67ea33c10 100644
--- a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceMessages.java
+++ b/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceMessages.java
@@ -56,11 +56,11 @@ public interface HadoopXmlResourceMessages {
@Message(level = MessageLevel.ERROR, text = "Parsing XML configuration {0} failed: {1}")
void failedToParseXmlConfiguration(String path, String errorMessage, @StackTrace(level = MessageLevel.DEBUG) Exception e);
- @Message(level = MessageLevel.DEBUG, text = "Processing Hadoop XML resource {0} (force = {1}; lastReloadTime = {2}; lastModified = {3})")
- void processHadoopXmlResource(String descriptorPath, boolean force, FileTime lastReloadTime, FileTime lastModifiedTime);
+ @Message(level = MessageLevel.DEBUG, text = "Processing Hadoop XML resource {0} (lastReloadTime = {1}; lastModified = {2})")
+ void processHadoopXmlResource(String descriptorPath, FileTime lastReloadTime, FileTime lastModifiedTime);
- @Message(level = MessageLevel.DEBUG, text = "Skipping Hadoop XML resource monitoring of {0} (force = {1}; lastReloadTime = {2}; lastModified = {3})")
- void skipMonitorHadoopXmlResource(String descriptorPath, boolean force, FileTime lastReloadTime, FileTime lastModifiedTime);
+ @Message(level = MessageLevel.DEBUG, text = "Skipping Hadoop XML resource monitoring of {0} (lastReloadTime = {1}; lastModified = {2})")
+ void skipMonitorHadoopXmlResource(String descriptorPath, FileTime lastReloadTime, FileTime lastModifiedTime);
@Message(level = MessageLevel.ERROR, text = "Error while monitoring Hadoop style XML configuration {0}: {1}")
void failedToMonitorHadoopXmlResource(String descriptorPath, String errorMessage, @StackTrace(level = MessageLevel.DEBUG) Exception e);
@@ -71,9 +71,6 @@ public interface HadoopXmlResourceMessages {
@Message(level = MessageLevel.ERROR, text = "Error while producing Knox provider: {0}")
void failedToProduceKnoxProvider(String errorMessage, @StackTrace(level = MessageLevel.DEBUG) Exception e);
- @Message(level = MessageLevel.WARN, text = "Service {0} is disabled. It will NOT be added in {1}")
- void serviceDisabled(String serviceName, String descriptorName);
-
@Message(level = MessageLevel.INFO, text = "Updated advanced service discovery configuration for {0}.")
void updatedAdvanceServiceDiscoverytConfiguration(String topologyName);
diff --git a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceMonitor.java b/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceMonitor.java
index f77d419f4..8ed729b60 100644
--- a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceMonitor.java
+++ b/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceMonitor.java
@@ -25,9 +25,8 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.FileTime;
import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@@ -36,19 +35,16 @@ import java.util.concurrent.locks.ReentrantLock;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.SuffixFileFilter;
-import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.concurrent.BasicThreadFactory;
import org.apache.knox.gateway.config.GatewayConfig;
import org.apache.knox.gateway.i18n.messages.MessagesFactory;
-import org.apache.knox.gateway.topology.discovery.advanced.AdvancedServiceDiscoveryConfig;
-import org.apache.knox.gateway.topology.discovery.advanced.AdvancedServiceDiscoveryConfigChangeListener;
import org.apache.knox.gateway.util.JsonUtils;
/**
* Monitoring KNOX_DESCRIPTOR_DIR for *.hxr files - which is a Hadoop XML configuration - and processing those files if they were modified
* since the last time it they were processed
*/
-public class HadoopXmlResourceMonitor implements AdvancedServiceDiscoveryConfigChangeListener {
+public class HadoopXmlResourceMonitor {
private static final String HADOOP_XML_RESOURCE_FILE_EXTENSION = ".hxr";
private static final HadoopXmlResourceMessages LOG = MessagesFactory.get(HadoopXmlResourceMessages.class);
@@ -72,31 +68,31 @@ public class HadoopXmlResourceMonitor implements AdvancedServiceDiscoveryConfigC
public void setupMonitor() {
if (monitoringInterval > 0) {
final ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(new BasicThreadFactory.Builder().namingPattern("ClouderaManagerDescriptorMonitor-%d").build());
- monitorClouderaManagerDescriptors(false); // call it explicitly first to generate descriptors up front, so that the health checker can pick them up
- executorService.scheduleAtFixedRate(() -> monitorClouderaManagerDescriptors(false), monitoringInterval, monitoringInterval, TimeUnit.MILLISECONDS);
+ monitorClouderaManagerDescriptors(); // call it explicitly first to generate descriptors up front, so that the health checker can pick them up
+ executorService.scheduleAtFixedRate(() -> monitorClouderaManagerDescriptors(), monitoringInterval, monitoringInterval, TimeUnit.MILLISECONDS);
LOG.monitoringHadoopXmlResources(descriptorsDir);
}
}
- private void monitorClouderaManagerDescriptors(boolean force) {
+ private void monitorClouderaManagerDescriptors() {
final File[] clouderaManagerDescriptorFiles = new File(descriptorsDir).listFiles((FileFilter) new SuffixFileFilter(HADOOP_XML_RESOURCE_FILE_EXTENSION));
for (File clouderaManagerDescriptorFile : clouderaManagerDescriptorFiles) {
- monitorClouderaManagerDescriptor(Paths.get(clouderaManagerDescriptorFile.getAbsolutePath()), force);
+ monitorClouderaManagerDescriptor(Paths.get(clouderaManagerDescriptorFile.getAbsolutePath()));
}
}
- private void monitorClouderaManagerDescriptor(Path clouderaManagerDescriptorFile, boolean force) {
+ private void monitorClouderaManagerDescriptor(Path clouderaManagerDescriptorFile) {
monitorLock.lock();
try {
if (Files.isReadable(clouderaManagerDescriptorFile)) {
final FileTime lastModifiedTime = Files.getLastModifiedTime(clouderaManagerDescriptorFile);
FileTime lastReloadTime = lastReloadTimes.get(clouderaManagerDescriptorFile);
- if (force || lastReloadTime == null || lastReloadTime.compareTo(lastModifiedTime) < 0) {
+ if (lastReloadTime == null || lastReloadTime.compareTo(lastModifiedTime) < 0) {
lastReloadTimes.put(clouderaManagerDescriptorFile, lastModifiedTime);
- LOG.processHadoopXmlResource(clouderaManagerDescriptorFile.toString(), force, lastReloadTime, lastModifiedTime);
+ LOG.processHadoopXmlResource(clouderaManagerDescriptorFile.toString(), lastReloadTime, lastModifiedTime);
processClouderaManagerDescriptor(clouderaManagerDescriptorFile.toString());
} else {
- LOG.skipMonitorHadoopXmlResource(clouderaManagerDescriptorFile.toString(), force, lastReloadTime, lastModifiedTime);
+ LOG.skipMonitorHadoopXmlResource(clouderaManagerDescriptorFile.toString(), lastReloadTime, lastModifiedTime);
}
} else {
LOG.failedToMonitorHadoopXmlResource(clouderaManagerDescriptorFile.toString(), "File is not readable!", null);
@@ -169,12 +165,4 @@ public class HadoopXmlResourceMonitor implements AdvancedServiceDiscoveryConfigC
return true;
}
- @Override
- public void onAdvancedServiceDiscoveryConfigurationChange(Properties newConfiguration) {
- final String topologyName = new AdvancedServiceDiscoveryConfig(newConfiguration).getTopologyName();
- if (StringUtils.isBlank(topologyName)) {
- throw new IllegalArgumentException("Invalid advanced service discovery configuration: topology name is missing!");
- }
- monitorClouderaManagerDescriptors(true);
- }
}
diff --git a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceParser.java b/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceParser.java
index a66c36361..75f00fe62 100644
--- a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceParser.java
+++ b/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceParser.java
@@ -23,20 +23,16 @@ import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Optional;
-import java.util.Properties;
import java.util.Set;
import java.util.TreeMap;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
-import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.knox.gateway.config.GatewayConfig;
import org.apache.knox.gateway.i18n.messages.MessagesFactory;
-import org.apache.knox.gateway.topology.discovery.advanced.AdvancedServiceDiscoveryConfig;
-import org.apache.knox.gateway.topology.discovery.advanced.AdvancedServiceDiscoveryConfigChangeListener;
import org.apache.knox.gateway.topology.simple.JSONProviderConfiguration;
import org.apache.knox.gateway.topology.simple.JSONProviderConfiguration.JSONProvider;
import org.apache.knox.gateway.topology.simple.ProviderConfiguration;
@@ -50,7 +46,7 @@ import org.apache.knox.gateway.topology.simple.SimpleDescriptorImpl.ServiceImpl;
* Parses Knox descriptors and provider configurations from Hadoop style XML config
*
*/
-public class HadoopXmlResourceParser implements AdvancedServiceDiscoveryConfigChangeListener {
+public class HadoopXmlResourceParser {
private static final HadoopXmlResourceMessages log = MessagesFactory.get(HadoopXmlResourceMessages.class);
//shared provider related constants
@@ -71,15 +67,14 @@ public class HadoopXmlResourceParser implements AdvancedServiceDiscoveryConfigCh
private static final String CONFIG_NAME_APPLICATION_PREFIX = "app";
private static final String CONFIG_NAME_SERVICE_URL = "url";
private static final String CONFIG_NAME_SERVICE_VERSION = "version";
+ private static final String CONFIG_NAME_SERVICES = "services";
private static final String REMOVE = "remove";
- private final Map<String, AdvancedServiceDiscoveryConfig> advancedServiceDiscoveryConfigMap;
private final String sharedProvidersDir;
private final GatewayConfig gatewayConfig;
public HadoopXmlResourceParser(GatewayConfig gatewayConfig) {
this.gatewayConfig = gatewayConfig;
- this.advancedServiceDiscoveryConfigMap = new ConcurrentHashMap<>();
this.sharedProvidersDir = gatewayConfig.getGatewayProvidersConfigDir();
}
@@ -270,6 +265,9 @@ public class HadoopXmlResourceParser implements AdvancedServiceDiscoveryConfigCh
case CONFIG_NAME_PROVISION_ENCRYPT_QUERY_STRING_CREDENTIAL:
descriptor.setProvisionEncryptQueryStringCredential(Boolean.valueOf(parameterPairParts[1].trim()));
break;
+ case CONFIG_NAME_SERVICES:
+ Arrays.asList(parameterPairParts[1].trim().split(",", -1)).forEach(serviceName -> descriptor.addService(emptyService(serviceName)));
+ break;
default:
if (parameterName.startsWith(CONFIG_NAME_APPLICATION_PREFIX)) {
parseApplication(descriptor, configurationPair.trim());
@@ -280,11 +278,6 @@ public class HadoopXmlResourceParser implements AdvancedServiceDiscoveryConfigCh
}
}
- final AdvancedServiceDiscoveryConfig advancedServiceDiscoveryConfig = advancedServiceDiscoveryConfigMap.get(name);
- if (advancedServiceDiscoveryConfig != null) {
- setDiscoveryDetails(descriptor, advancedServiceDiscoveryConfig);
- addEnabledServices(descriptor, advancedServiceDiscoveryConfig);
- }
return descriptor;
} catch (Exception e) {
log.failedToParseDescriptor(name, e.getMessage(), e);
@@ -292,31 +285,10 @@ public class HadoopXmlResourceParser implements AdvancedServiceDiscoveryConfigCh
}
}
- private void setDiscoveryDetails(SimpleDescriptorImpl descriptor, AdvancedServiceDiscoveryConfig advancedServiceDiscoveryConfig) {
- if (StringUtils.isBlank(descriptor.getDiscoveryAddress())) {
- descriptor.setDiscoveryAddress(advancedServiceDiscoveryConfig.getDiscoveryAddress());
- }
-
- if (StringUtils.isBlank(descriptor.getCluster())) {
- descriptor.setCluster(advancedServiceDiscoveryConfig.getDiscoveryCluster());
- }
-
- if (StringUtils.isBlank(descriptor.getDiscoveryType())) {
- descriptor.setDiscoveryType("ClouderaManager");
- }
- }
-
- /*
- * Adds any enabled service which is not listed in the CM descriptor
- */
- private void addEnabledServices(SimpleDescriptorImpl descriptor, AdvancedServiceDiscoveryConfig advancedServiceDiscoveryConfig) {
- advancedServiceDiscoveryConfig.getEnabledServiceNames().forEach(enabledServiceName -> {
- if (descriptor.getService(enabledServiceName) == null) {
- ServiceImpl service = new ServiceImpl();
- service.setName(enabledServiceName);
- descriptor.addService(service);
- }
- });
+ private ServiceImpl emptyService(String serviceName) {
+ final ServiceImpl service = new ServiceImpl();
+ service.setName(serviceName.trim().toUpperCase(Locale.ROOT));
+ return service;
}
/**
@@ -354,7 +326,7 @@ public class HadoopXmlResourceParser implements AdvancedServiceDiscoveryConfigCh
* <li><code>$SERVICE_NAME:version=$VERSION</code> (optional)</li>
* <li><code>$SERVICE_NAME[:$PARAMETER_NAME=$PARAMETER_VALUE] (optional)</code></li>
* </ul>
- * Sample application configurations:
+ * Sample service configurations:
* <ul>
* <li>HIVE:url=http://localhost:123</li>
* <li>HIVE:version=1.0</li>
@@ -364,50 +336,31 @@ public class HadoopXmlResourceParser implements AdvancedServiceDiscoveryConfigCh
private void parseService(SimpleDescriptorImpl descriptor, String configurationPair) {
final String[] serviceParts = configurationPair.split(":");
final String serviceName = serviceParts[0].trim();
- if (isServiceEnabled(descriptor.getName(), serviceName)) {
- ServiceImpl service = (ServiceImpl) descriptor.getService(serviceName);
- if (service == null) {
- service = new ServiceImpl();
- service.setName(serviceName);
- descriptor.addService(service);
- }
- if (serviceParts.length > 1) {
- // configuration value may contain ":" (for instance http://host:port) -> considering a configuration name/value pair everything after '$SERVICE_NAME:'
- final String serviceConfiguration = configurationPair.substring(serviceName.length() + 1).trim();
- final String[] serviceConfigurationParts = serviceConfiguration.split("=", 2);
- final String serviceConfigurationName = serviceConfigurationParts[0].trim();
- final String serviceConfigurationValue = serviceConfigurationParts[1].trim();
- switch (serviceConfigurationName) {
- case CONFIG_NAME_SERVICE_URL:
- service.addUrl(serviceConfigurationValue);
- break;
- case CONFIG_NAME_SERVICE_VERSION:
- service.setVersion(serviceConfigurationValue);
- break;
- default:
- service.addParam(serviceConfigurationName, serviceConfigurationValue);
- break;
- }
- }
- } else {
- log.serviceDisabled(serviceName, descriptor.getName());
+ ServiceImpl service = (ServiceImpl) descriptor.getService(serviceName);
+ if (service == null) {
+ service = emptyService(serviceName);
+ descriptor.addService(service);
}
- }
- private boolean isServiceEnabled(String descriptorName, String serviceName) {
- return advancedServiceDiscoveryConfigMap.containsKey(descriptorName) ? advancedServiceDiscoveryConfigMap.get(descriptorName).isServiceEnabled(serviceName) : true;
- }
-
- @Override
- public void onAdvancedServiceDiscoveryConfigurationChange(Properties newConfiguration) {
- final AdvancedServiceDiscoveryConfig advancedServiceDiscoveryConfig = new AdvancedServiceDiscoveryConfig(newConfiguration);
- final String topologyName = advancedServiceDiscoveryConfig.getTopologyName();
- if (StringUtils.isBlank(topologyName)) {
- throw new IllegalArgumentException("Invalid advanced service discovery configuration: topology name is missing!");
+ if (serviceParts.length > 1) {
+ // configuration value may contain ":" (for instance http://host:port) -> considering a configuration name/value pair everything after '$SERVICE_NAME:'
+ final String serviceConfiguration = configurationPair.substring(serviceName.length() + 1).trim();
+ final String[] serviceConfigurationParts = serviceConfiguration.split("=", 2);
+ final String serviceConfigurationName = serviceConfigurationParts[0].trim();
+ final String serviceConfigurationValue = serviceConfigurationParts[1].trim();
+ switch (serviceConfigurationName) {
+ case CONFIG_NAME_SERVICE_URL:
+ service.addUrl(serviceConfigurationValue);
+ break;
+ case CONFIG_NAME_SERVICE_VERSION:
+ service.setVersion(serviceConfigurationValue);
+ break;
+ default:
+ service.addParam(serviceConfigurationName, serviceConfigurationValue);
+ break;
+ }
}
- advancedServiceDiscoveryConfigMap.put(topologyName, advancedServiceDiscoveryConfig);
- log.updatedAdvanceServiceDiscoverytConfiguration(topologyName);
}
}
diff --git a/gateway-topology-hadoop-xml/src/test/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceParserTest.java b/gateway-topology-hadoop-xml/src/test/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceParserTest.java
index a0e78f619..3f80d91c4 100644
--- a/gateway-topology-hadoop-xml/src/test/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceParserTest.java
+++ b/gateway-topology-hadoop-xml/src/test/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceParserTest.java
@@ -33,14 +33,12 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.Properties;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.io.FileUtils;
import org.apache.knox.gateway.config.GatewayConfig;
-import org.apache.knox.gateway.topology.discovery.advanced.AdvancedServiceDiscoveryConfig;
import org.apache.knox.gateway.topology.simple.ProviderConfiguration;
import org.apache.knox.gateway.topology.simple.SimpleDescriptor;
import org.apache.knox.gateway.topology.simple.SimpleDescriptor.Application;
@@ -129,50 +127,26 @@ public class HadoopXmlResourceParserTest {
}
@Test
- public void testCMDescriptorParserWithNotEnabledServices() throws Exception {
- final String testConfigPath = this.getClass().getClassLoader().getResource("testDescriptor.xml").getPath();
-
- final Properties advancedConfigurationTopology1 = new Properties();
- advancedConfigurationTopology1.put(buildEnabledParameter("topology1", "HIVE"), "false");
- advancedConfigurationTopology1.put(AdvancedServiceDiscoveryConfig.PARAMETER_NAME_TOPOLOGY_NAME, "topology1");
- hadoopXmlResourceParser.onAdvancedServiceDiscoveryConfigurationChange(advancedConfigurationTopology1);
-
- final Properties advancedConfigurationTopology2 = new Properties();
- advancedConfigurationTopology2.put(buildEnabledParameter("topology2", "NIFI"), "false");
- advancedConfigurationTopology2.put(AdvancedServiceDiscoveryConfig.PARAMETER_NAME_TOPOLOGY_NAME, "topology2");
- hadoopXmlResourceParser.onAdvancedServiceDiscoveryConfigurationChange(advancedConfigurationTopology2);
+ public void testDescriptorParserWithServices() throws Exception {
+ final String testConfigPath = this.getClass().getClassLoader().getResource("testDescriptorWithServiceList.xml").getPath();
final Set<SimpleDescriptor> descriptors = hadoopXmlResourceParser.parse(testConfigPath).getDescriptors();
- assertEquals(2, descriptors.size());
+ assertEquals(1, descriptors.size());
final Iterator<SimpleDescriptor> descriptorsIterator = descriptors.iterator();
- SimpleDescriptor topology1 = descriptorsIterator.next();
+ final SimpleDescriptor topology1 = descriptorsIterator.next();
assertNotNull(topology1);
- // topology1 comes with HIVE which is disabled
- assertTrue(topology1.getServices().isEmpty());
-
- SimpleDescriptor topology2 = descriptorsIterator.next();
- assertNotNull(topology2);
- // topology1 comes with ATLAS and NIFI but the latter one is disabled
- validateTopology2Descriptors(topology2, false);
- }
-
- @Test
- public void testCMDescriptorParserWithEnabledNotListedServiceInTopology1() throws Exception {
- final String testConfigPath = this.getClass().getClassLoader().getResource("testDescriptor.xml").getPath();
- final Properties advancedConfiguration = new Properties();
- advancedConfiguration.put(buildEnabledParameter("topology1", "oozie"), "true"); //it should not matter if service name is lowercase advanced configuration
- advancedConfiguration.put(AdvancedServiceDiscoveryConfig.PARAMETER_NAME_TOPOLOGY_NAME, "topology1");
- hadoopXmlResourceParser.onAdvancedServiceDiscoveryConfigurationChange(advancedConfiguration);
- final Set<SimpleDescriptor> descriptors = hadoopXmlResourceParser.parse(testConfigPath).getDescriptors();
- final Iterator<SimpleDescriptor> descriptorsIterator = descriptors.iterator();
- SimpleDescriptor descriptor = descriptorsIterator.next();
- assertNotNull(descriptor);
- // topology1 comes without OOZIE but it's enabled in topology1 -> OOZIE should be added without any url/version/parameter
- assertService(descriptor, "OOZIE", null, null, null);
-
- descriptor = descriptorsIterator.next();
- validateTopology2Descriptors(descriptor, true);
- assertNull(descriptor.getService("OOZIE"));
+ // services= NIFI,ATLAS ,hive, hue ,IMPALA#
+ // + RANGER and FLINK service declarations in separate lines
+ assertEquals(7, topology1.getServices().size());
+ assertNotNull(topology1.getService("NIFI"));
+ assertNotNull(topology1.getService("ATLAS"));
+ assertNotNull(topology1.getService("HIVE")); //should be uppercase
+ assertNotNull(topology1.getService("HUE")); //should be uppercase
+ assertNotNull(topology1.getService("IMPALA"));
+ assertNotNull(topology1.getService("RANGER"));
+ assertNotNull(topology1.getService("FLINK"));
+ assertEquals("3.0", topology1.getService("FLINK").getVersion());
+ assertNull(topology1.getService("AMBARI"));
}
@Test
@@ -219,28 +193,6 @@ public class HadoopXmlResourceParserTest {
validateTestDescriptorProviderConfigs(parserResult.getProviders(), "ldap://localhost:33389", true, false);
}
- private String buildEnabledParameter(String topologyName, String serviceName) {
- return AdvancedServiceDiscoveryConfig.PARAMETER_NAME_PREFIX_ENABLED_SERVICE + topologyName + AdvancedServiceDiscoveryConfig.PARAMETER_NAME_POSTFIX_ENABLED_SERVICE + serviceName;
- }
-
- @Test
- public void testSettingDiscoveryDetails() throws Exception {
- final String address = "http://myCmHost:7180";
- final String cluster = "My Test Cluster";
- final String testConfigPath = this.getClass().getClassLoader().getResource("testDescriptorWithoutDiscoveryDetails.xml").getPath();
- final Properties advancedConfiguration = new Properties();
- advancedConfiguration.put(AdvancedServiceDiscoveryConfig.PARAMETER_NAME_TOPOLOGY_NAME, "topology1");
- advancedConfiguration.put(AdvancedServiceDiscoveryConfig.PARAMETER_NAME_DISCOVERY_ADDRESS, address);
- advancedConfiguration.put(AdvancedServiceDiscoveryConfig.PARAMETER_NAME_DISCOVERY_CLUSTER, cluster);
- hadoopXmlResourceParser.onAdvancedServiceDiscoveryConfigurationChange(advancedConfiguration);
- final Set<SimpleDescriptor> descriptors = hadoopXmlResourceParser.parse(testConfigPath).getDescriptors();
- final Iterator<SimpleDescriptor> descriptorsIterator = descriptors.iterator();
- SimpleDescriptor descriptor = descriptorsIterator.next();
- assertEquals(address, descriptor.getDiscoveryAddress());
- assertEquals(cluster, descriptor.getCluster());
- assertEquals("ClouderaManager", descriptor.getDiscoveryType());
- }
-
@Test
public void testDelete() throws Exception {
String testConfigPath = this.getClass().getClassLoader().getResource("testDelete.xml").getPath();
diff --git a/gateway-topology-hadoop-xml/src/test/resources/testDescriptorWithServiceList.xml b/gateway-topology-hadoop-xml/src/test/resources/testDescriptorWithServiceList.xml
new file mode 100644
index 000000000..978133631
--- /dev/null
+++ b/gateway-topology-hadoop-xml/src/test/resources/testDescriptorWithServiceList.xml
@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<configuration>
+ <property>
+ <name>topology1</name>
+ <value>
+ discoveryType=ClouderaManager#
+ discoveryAddress=http://host:123#
+ discoveryUser=user#
+ discoveryPasswordAlias=alias#
+ cluster=Cluster 1#
+ providerConfigRef=topology1-provider#
+ app:knoxauth:param1.name=param1.value#
+ app:admin-ui#
+ services=NIFI,ATLAS,hive,hue,IMPALA#
+ RANGER#
+ FLINK:version=3.0
+ </value>
+ </property>
+</configuration>
\ No newline at end of file