You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@knox.apache.org by sm...@apache.org on 2023/07/24 08:41:38 UTC
[knox] branch master updated: KNOX-2942 - Miscellaneous HXR parser improvements (#779)
This is an automated email from the ASF dual-hosted git repository.
smolnar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/knox.git
The following commit(s) were added to refs/heads/master by this push:
new 8c4db7a9c KNOX-2942 - Miscellaneous HXR parser improvements (#779)
8c4db7a9c is described below
commit 8c4db7a9c46dbb9740141240c5ad943defdddc72
Author: Sandor Molnar <sm...@apache.org>
AuthorDate: Mon Jul 24 10:41:33 2023 +0200
KNOX-2942 - Miscellaneous HXR parser improvements (#779)
---
.../org/apache/knox/gateway/GatewayServer.java | 6 ++--
.../hadoop/xml/HadoopXmlResourceMessages.java | 12 +++++--
.../hadoop/xml/HadoopXmlResourceMonitor.java | 37 ++++++++++++++--------
.../hadoop/xml/HadoopXmlResourceParser.java | 24 ++++----------
.../hadoop/xml/HadoopXmlResourceParserTest.java | 8 -----
5 files changed, 42 insertions(+), 45 deletions(-)
diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/GatewayServer.java b/gateway-server/src/main/java/org/apache/knox/gateway/GatewayServer.java
index 65fefd50d..8485ec86d 100644
--- a/gateway-server/src/main/java/org/apache/knox/gateway/GatewayServer.java
+++ b/gateway-server/src/main/java/org/apache/knox/gateway/GatewayServer.java
@@ -625,10 +625,10 @@ public class GatewayServer {
File topologiesDir = calculateAbsoluteTopologiesDir();
monitor = services.getService(ServiceType.TOPOLOGY_SERVICE);
- // Descriptors should be reloaded before topology reloading at startup, so that any changes to descriptors
+ // Shared providers and descriptors should be reloaded before topology reloading at startup, so that any changes to descriptors
// will be realized before Knox deploys "old" topologies that would have re-deployed anyway in a matter of seconds
// by the descriptor monitor
- monitor.reloadDescriptors();
+ handleHadoopXmlResources();
monitor.addTopologyChangeListener(listener);
log.loadingTopologiesFromDirectory(topologiesDir.getAbsolutePath());
@@ -700,8 +700,6 @@ public class GatewayServer {
cleanupTopologyDeployments();
- handleHadoopXmlResources();
-
// Start the topology monitor.
monitor.startMonitor();
diff --git a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceMessages.java b/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceMessages.java
index 6390e7f5b..63d92dfcb 100644
--- a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceMessages.java
+++ b/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceMessages.java
@@ -16,6 +16,8 @@
*/
package org.apache.knox.gateway.topology.hadoop.xml;
+import java.nio.file.attribute.FileTime;
+
import org.apache.knox.gateway.i18n.messages.Message;
import org.apache.knox.gateway.i18n.messages.MessageLevel;
import org.apache.knox.gateway.i18n.messages.Messages;
@@ -30,8 +32,8 @@ public interface HadoopXmlResourceMessages {
@Message(level = MessageLevel.INFO, text = "Monitoring Knox resources in Hadoop style XML configurations is disabled.")
void disableMonitoringHadoopXmlResources();
- @Message(level = MessageLevel.INFO, text = "Parsing Knox resources in Hadoop style XML {0}. Looking up {1}...")
- void parseHadoopXmlResource(String path, String topologyName);
+ @Message(level = MessageLevel.INFO, text = "Parsing Knox resources in Hadoop style XML {0}...")
+ void parseHadoopXmlResource(String path);
@Message(level = MessageLevel.INFO, text = "Found Knox descriptors {0} in {1}")
void foundKnoxDescriptors(String descriptorList, String path);
@@ -54,6 +56,12 @@ public interface HadoopXmlResourceMessages {
@Message(level = MessageLevel.ERROR, text = "Parsing XML configuration {0} failed: {1}")
void failedToParseXmlConfiguration(String path, String errorMessage, @StackTrace(level = MessageLevel.DEBUG) Exception e);
+ @Message(level = MessageLevel.DEBUG, text = "Processing Hadoop XML resource {0} (force = {1}; lastReloadTime = {2}; lastModified = {3})")
+ void processHadoopXmlResource(String descriptorPath, boolean force, FileTime lastReloadTime, FileTime lastModifiedTime);
+
+ @Message(level = MessageLevel.DEBUG, text = "Skipping Hadoop XML resource monitoring of {0} (force = {1}; lastReloadTime = {2}; lastModified = {3})")
+ void skipMonitorHadoopXmlResource(String descriptorPath, boolean force, FileTime lastReloadTime, FileTime lastModifiedTime);
+
@Message(level = MessageLevel.ERROR, text = "Error while monitoring Hadoop style XML configuration {0}: {1}")
void failedToMonitorHadoopXmlResource(String descriptorPath, String errorMessage, @StackTrace(level = MessageLevel.DEBUG) Exception e);
diff --git a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceMonitor.java b/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceMonitor.java
index 674724fe0..79c494e15 100644
--- a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceMonitor.java
+++ b/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceMonitor.java
@@ -24,11 +24,15 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.FileTime;
+import java.util.Map;
import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.SuffixFileFilter;
@@ -52,50 +56,57 @@ public class HadoopXmlResourceMonitor implements AdvancedServiceDiscoveryConfigC
private final String descriptorsDir;
private final long monitoringInterval;
private final HadoopXmlResourceParser hadoopXmlResourceParser;
- private FileTime lastReloadTime;
+ private final Map<Path, FileTime> lastReloadTimes;
+ private final Lock monitorLock = new ReentrantLock();
public HadoopXmlResourceMonitor(GatewayConfig gatewayConfig, HadoopXmlResourceParser hadoopXmlResourceParser) {
this.hadoopXmlResourceParser = hadoopXmlResourceParser;
this.sharedProvidersDir = gatewayConfig.getGatewayProvidersConfigDir();
this.descriptorsDir = gatewayConfig.getGatewayDescriptorsDir();
this.monitoringInterval = gatewayConfig.getClouderaManagerDescriptorsMonitoringInterval();
+ this.lastReloadTimes = new ConcurrentHashMap<>();
}
public void setupMonitor() {
if (monitoringInterval > 0) {
final ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(new BasicThreadFactory.Builder().namingPattern("ClouderaManagerDescriptorMonitor-%d").build());
- executorService.scheduleAtFixedRate(() -> monitorClouderaManagerDescriptors(null), 0, monitoringInterval, TimeUnit.MILLISECONDS);
+ executorService.scheduleAtFixedRate(() -> monitorClouderaManagerDescriptors(false), 0, monitoringInterval, TimeUnit.MILLISECONDS);
LOG.monitoringHadoopXmlResources(descriptorsDir);
- } else {
- LOG.disableMonitoringHadoopXmlResources();
}
}
- private void monitorClouderaManagerDescriptors(String topologyName) {
+ private void monitorClouderaManagerDescriptors(boolean force) {
final File[] clouderaManagerDescriptorFiles = new File(descriptorsDir).listFiles((FileFilter) new SuffixFileFilter(HADOOP_XML_RESOURCE_FILE_EXTENSION));
for (File clouderaManagerDescriptorFile : clouderaManagerDescriptorFiles) {
- monitorClouderaManagerDescriptor(Paths.get(clouderaManagerDescriptorFile.getAbsolutePath()), topologyName);
+ monitorClouderaManagerDescriptor(Paths.get(clouderaManagerDescriptorFile.getAbsolutePath()), force);
}
}
- private void monitorClouderaManagerDescriptor(Path clouderaManagerDescriptorFile, String topologyName) {
+ private void monitorClouderaManagerDescriptor(Path clouderaManagerDescriptorFile, boolean force) {
+ monitorLock.lock();
try {
if (Files.isReadable(clouderaManagerDescriptorFile)) {
final FileTime lastModifiedTime = Files.getLastModifiedTime(clouderaManagerDescriptorFile);
- if (topologyName != null || lastReloadTime == null || lastReloadTime.compareTo(lastModifiedTime) < 0) {
- lastReloadTime = lastModifiedTime;
- processClouderaManagerDescriptor(clouderaManagerDescriptorFile.toString(), topologyName);
+ FileTime lastReloadTime = lastReloadTimes.get(clouderaManagerDescriptorFile);
+ if (force || lastReloadTime == null || lastReloadTime.compareTo(lastModifiedTime) < 0) {
+ lastReloadTimes.put(clouderaManagerDescriptorFile, lastModifiedTime);
+ LOG.processHadoopXmlResource(clouderaManagerDescriptorFile.toString(), force, lastReloadTime, lastModifiedTime);
+ processClouderaManagerDescriptor(clouderaManagerDescriptorFile.toString());
+ } else {
+ LOG.skipMonitorHadoopXmlResource(clouderaManagerDescriptorFile.toString(), force, lastReloadTime, lastModifiedTime);
}
} else {
LOG.failedToMonitorHadoopXmlResource(clouderaManagerDescriptorFile.toString(), "File is not readable!", null);
}
} catch (IOException e) {
LOG.failedToMonitorHadoopXmlResource(clouderaManagerDescriptorFile.toString(), e.getMessage(), e);
+ } finally {
+ monitorLock.unlock();
}
}
- private void processClouderaManagerDescriptor(String descriptorFilePath, String topologyName) {
- final HadoopXmlResourceParserResult result = hadoopXmlResourceParser.parse(descriptorFilePath, topologyName);
+ private void processClouderaManagerDescriptor(String descriptorFilePath) {
+ final HadoopXmlResourceParserResult result = hadoopXmlResourceParser.parse(descriptorFilePath);
processSharedProviders(result);
processDescriptors(result);
processDeleted(descriptorsDir, result.getDeletedDescriptors(), ".json");
@@ -160,6 +171,6 @@ public class HadoopXmlResourceMonitor implements AdvancedServiceDiscoveryConfigC
if (StringUtils.isBlank(topologyName)) {
throw new IllegalArgumentException("Invalid advanced service discovery configuration: topology name is missing!");
}
- monitorClouderaManagerDescriptors(topologyName);
+ monitorClouderaManagerDescriptors(true);
}
}
diff --git a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceParser.java b/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceParser.java
index d00d358fc..4e1dfce60 100644
--- a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceParser.java
+++ b/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceParser.java
@@ -82,33 +82,21 @@ public class HadoopXmlResourceParser implements AdvancedServiceDiscoveryConfigCh
this.sharedProvidersDir = gatewayConfig.getGatewayProvidersConfigDir();
}
- /**
- * Produces a set of {@link SimpleDescriptor}s from the specified file. Parses ALL descriptors listed in the given file.
- *
- * @param path
- * The path to the configuration file which holds descriptor information in a pre-defined format.
- * @return A SimpleDescriptor based on the contents of the given file.
- */
- public HadoopXmlResourceParserResult parse(String path) {
- return parse(path, null);
- }
-
/**
* Produces a set of {@link SimpleDescriptor}s from the specified file.
*
* @param path
* The path to the configuration file which holds descriptor information in a pre-defined format.
- * @param topologyName
* if set, the parser should only parse a descriptor with the same name
* @return A SimpleDescriptor based on the contents of the given file.
*/
- public HadoopXmlResourceParserResult parse(String path, String topologyName) {
+ public HadoopXmlResourceParserResult parse(String path) {
try {
- log.parseHadoopXmlResource(path, topologyName == null ? "all topologies" : topologyName);
+ log.parseHadoopXmlResource(path);
final Configuration xmlConfiguration = new Configuration(false);
xmlConfiguration.addResource(Paths.get(path).toUri().toURL());
xmlConfiguration.reloadConfiguration();
- final HadoopXmlResourceParserResult parserResult = parseXmlConfig(xmlConfiguration, topologyName);
+ final HadoopXmlResourceParserResult parserResult = parseXmlConfig(xmlConfiguration);
logParserResult(path, parserResult);
return parserResult;
} catch (Exception e) {
@@ -132,7 +120,7 @@ public class HadoopXmlResourceParser implements AdvancedServiceDiscoveryConfigCh
}
}
- private HadoopXmlResourceParserResult parseXmlConfig(Configuration xmlConfiguration, String topologyName) {
+ private HadoopXmlResourceParserResult parseXmlConfig(Configuration xmlConfiguration) {
final Map<String, ProviderConfiguration> providers = new LinkedHashMap<>();
final Set<SimpleDescriptor> descriptors = new LinkedHashSet<>();
Set<String> deletedDescriptors = new HashSet<>();
@@ -143,8 +131,8 @@ public class HadoopXmlResourceParser implements AdvancedServiceDiscoveryConfigCh
final String[] providerConfigurations = xmlConfigurationKey.replace(CONFIG_NAME_PROVIDER_CONFIGS_PREFIX, "").split(",");
Arrays.stream(providerConfigurations).map(String::trim).forEach(providerConfigurationName ->
parseProvider(providerConfigurationName, xmlDescriptor.getValue(), providers, deletedProviders));
- } else if (topologyName == null || xmlConfigurationKey.equals(topologyName)) {
- parseDescriptor(xmlConfigurationKey, xmlDescriptor.getValue(), descriptors, deletedDescriptors);
+ } else {
+ parseDescriptor(xmlConfigurationKey, xmlDescriptor.getValue(), descriptors, deletedDescriptors);
}
});
return new HadoopXmlResourceParserResult(providers, descriptors, deletedDescriptors, deletedProviders);
diff --git a/gateway-topology-hadoop-xml/src/test/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceParserTest.java b/gateway-topology-hadoop-xml/src/test/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceParserTest.java
index 1ee49cbcd..bca77c815 100644
--- a/gateway-topology-hadoop-xml/src/test/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceParserTest.java
+++ b/gateway-topology-hadoop-xml/src/test/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceParserTest.java
@@ -111,14 +111,6 @@ public class HadoopXmlResourceParserTest {
assertNotNull(parserResult.getProviders().get("admin"));
}
- @Test
- public void testCMDescriptorParserOnlyTopology2() throws Exception {
- final String testConfigPath = this.getClass().getClassLoader().getResource("testDescriptor.xml").getPath();
- final Set<SimpleDescriptor> descriptors = hadoopXmlResourceParser.parse(testConfigPath, "topology2").getDescriptors();
- assertEquals(1, descriptors.size());
- validateTopology2Descriptors(descriptors.iterator().next(), true);
- }
-
@Test
public void testCMDescriptorParserWrongDescriptorContent() throws Exception {
final String testConfigPath = this.getClass().getClassLoader().getResource("testDescriptorConfigurationWithWrongDescriptor.xml").getPath();