You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@knox.apache.org by am...@apache.org on 2023/04/26 08:10:13 UTC
[knox] branch master updated: KNOX-2901 - Deleting a descriptor/provider via hadoop xml resource (#750)
This is an automated email from the ASF dual-hosted git repository.
amagyar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/knox.git
The following commit(s) were added to refs/heads/master by this push:
new b2bc33f3b KNOX-2901 - Deleting a descriptor/provider via hadoop xml resource (#750)
b2bc33f3b is described below
commit b2bc33f3ba02e23066baebbdd2f6dde160ecec5d
Author: Attila Magyar <m....@gmail.com>
AuthorDate: Wed Apr 26 10:10:06 2023 +0200
KNOX-2901 - Deleting a descriptor/provider via hadoop xml resource (#750)
---
.../hadoop/xml/HadoopXmlResourceMessages.java | 9 +++
.../hadoop/xml/HadoopXmlResourceMonitor.java | 13 ++++
.../hadoop/xml/HadoopXmlResourceParser.java | 90 ++++++++++++++--------
.../hadoop/xml/HadoopXmlResourceParserResult.java | 16 +++-
.../hadoop/xml/HadoopXmlResourceParserTest.java | 10 +++
.../src/test/resources/testDelete.xml | 66 ++++++++++++++++
6 files changed, 170 insertions(+), 34 deletions(-)
diff --git a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceMessages.java b/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceMessages.java
index b849c5823..6390e7f5b 100644
--- a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceMessages.java
+++ b/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceMessages.java
@@ -74,4 +74,13 @@ public interface HadoopXmlResourceMessages {
@Message(level = MessageLevel.WARN, text = "Skipping read only provider: {0}.")
void skipReadOnlyProvider(String key);
+
+ @Message(level = MessageLevel.INFO, text = "Found deleted descriptors {0} in {1}")
+ void foundKnoxDeletedDescriptors(String descriptorList, String path);
+
+ @Message(level = MessageLevel.INFO, text = "Found deleted provider configurations {0} in {1}")
+ void foundKnoxDeletedProviderConfigurations(String providerConfigurationList, String path);
+
+ @Message(level = MessageLevel.INFO, text = "Deleting file {0}")
+ void deleteFile(String name);
}
diff --git a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceMonitor.java b/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceMonitor.java
index 00ed45737..674724fe0 100644
--- a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceMonitor.java
+++ b/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceMonitor.java
@@ -25,6 +25,7 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.FileTime;
import java.util.Properties;
+import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@@ -97,6 +98,18 @@ public class HadoopXmlResourceMonitor implements AdvancedServiceDiscoveryConfigC
final HadoopXmlResourceParserResult result = hadoopXmlResourceParser.parse(descriptorFilePath, topologyName);
processSharedProviders(result);
processDescriptors(result);
+ processDeleted(descriptorsDir, result.getDeletedDescriptors(), ".json");
+ processDeleted(sharedProvidersDir, result.getDeletedProviders(), ".json");
+ }
+
+ private void processDeleted(String parentDirectory, Set<String> deletedFileNames, String extension) {
+ for (String each : deletedFileNames) {
+ File fileToBeDeleted = new File(parentDirectory, each + extension);
+ if (fileToBeDeleted.exists()) {
+ LOG.deleteFile(fileToBeDeleted.getAbsolutePath());
+ fileToBeDeleted.delete();
+ }
+ }
}
private void processSharedProviders(final HadoopXmlResourceParserResult result) {
diff --git a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceParser.java b/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceParser.java
index 3ce78ff17..ec31b2c9d 100644
--- a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceParser.java
+++ b/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceParser.java
@@ -19,6 +19,7 @@ package org.apache.knox.gateway.topology.hadoop.xml;
import java.io.File;
import java.nio.file.Paths;
import java.util.Arrays;
+import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
@@ -58,7 +59,6 @@ public class HadoopXmlResourceParser implements AdvancedServiceDiscoveryConfigCh
private static final String CONFIG_NAME_PROVIDER_CONFIGS_NAME_PREFIX = "name=";
private static final String CONFIG_NAME_PROVIDER_CONFIGS_ENABLED_PREFIX = "enabled=";
private static final String CONFIG_NAME_PROVIDER_CONFIGS_PARAM_PREFIX = "param.";
- private static final String CONFIG_NAME_PROVIDER_CONFIGS_PARAM_REMOVE = "remove";
//descriptor related constants
private static final String CONFIG_NAME_DISCOVERY_TYPE = "discoveryType";
@@ -70,6 +70,7 @@ public class HadoopXmlResourceParser implements AdvancedServiceDiscoveryConfigCh
private static final String CONFIG_NAME_APPLICATION_PREFIX = "app";
private static final String CONFIG_NAME_SERVICE_URL = "url";
private static final String CONFIG_NAME_SERVICE_VERSION = "version";
+ private static final String REMOVE = "remove";
private final Map<String, AdvancedServiceDiscoveryConfig> advancedServiceDiscoveryConfigMap;
private final String sharedProvidersDir;
@@ -118,49 +119,74 @@ public class HadoopXmlResourceParser implements AdvancedServiceDiscoveryConfigCh
private void logParserResult(String path, final HadoopXmlResourceParserResult parserResult) {
if (!parserResult.getDescriptors().isEmpty()) {
- log.foundKnoxDescriptors(String.join(", ", parserResult.getDescriptors().stream().map(descriptor -> descriptor.getName()).collect(Collectors.toSet())), path);
+ log.foundKnoxDescriptors(String.join(", ", parserResult.getDescriptors().stream().map(SimpleDescriptor::getName).collect(Collectors.toSet())), path);
}
if (!parserResult.getProviders().isEmpty()) {
- log.foundKnoxProviderConfigurations(String.join(", ", parserResult.getProviders().keySet().stream().collect(Collectors.toSet())), path);
+ log.foundKnoxProviderConfigurations(String.join(", ", new HashSet<>(parserResult.getProviders().keySet())), path);
+ }
+ if (!parserResult.getDeletedDescriptors().isEmpty()) {
+ log.foundKnoxDeletedDescriptors(String.join(", ", parserResult.getDeletedDescriptors()), path);
+ }
+ if (!parserResult.getDeletedProviders().isEmpty()) {
+ log.foundKnoxDeletedProviderConfigurations(String.join(", ", parserResult.getDeletedProviders()), path);
}
}
private HadoopXmlResourceParserResult parseXmlConfig(Configuration xmlConfiguration, String topologyName) {
final Map<String, ProviderConfiguration> providers = new LinkedHashMap<>();
final Set<SimpleDescriptor> descriptors = new LinkedHashSet<>();
+ Set<String> deletedDescriptors = new HashSet<>();
+ Set<String> deletedProviders = new HashSet<>();
xmlConfiguration.forEach(xmlDescriptor -> {
String xmlConfigurationKey = xmlDescriptor.getKey();
if (xmlConfigurationKey.startsWith(CONFIG_NAME_PROVIDER_CONFIGS_PREFIX)) {
final String[] providerConfigurations = xmlConfigurationKey.replace(CONFIG_NAME_PROVIDER_CONFIGS_PREFIX, "").split(",");
- Arrays.asList(providerConfigurations).stream().map(providerConfigurationName -> providerConfigurationName.trim()).forEach(providerConfigurationName -> {
- if (gatewayConfig.getReadOnlyOverrideProviderNames().contains(providerConfigurationName)) {
- log.skipReadOnlyProvider(providerConfigurationName);
- } else {
- final File providerConfigFile = resolveProviderConfiguration(providerConfigurationName);
- try {
- final ProviderConfiguration providerConfiguration = getProviderConfiguration(providers, providerConfigFile, providerConfigurationName);
- providerConfiguration.setReadOnly(true);
- providerConfiguration.saveOrUpdateProviders(parseProviderConfigurations(xmlDescriptor.getValue(), providerConfiguration));
- providers.put(providerConfigurationName, providerConfiguration);
- } catch (Exception e) {
- log.failedToParseProviderConfiguration(providerConfigurationName, e.getMessage(), e);
- }
- }
- });
- } else {
- if (topologyName == null || xmlConfigurationKey.equals(topologyName)) {
- if (gatewayConfig.getReadOnlyOverrideTopologyNames().contains(xmlConfigurationKey)) {
- log.skipReadOnlyDescriptor(xmlConfigurationKey);
- } else {
- SimpleDescriptor descriptor = parseXmlDescriptor(xmlConfigurationKey, xmlDescriptor.getValue());
- if (descriptor != null) {
- descriptors.add(descriptor);
- }
- }
- }
+ Arrays.stream(providerConfigurations).map(String::trim).forEach(providerConfigurationName ->
+ parseProvider(providerConfigurationName, xmlDescriptor.getValue(), providers, deletedProviders));
+ } else if (topologyName == null || xmlConfigurationKey.equals(topologyName)) {
+ parseDescriptor(xmlConfigurationKey, xmlDescriptor.getValue(), descriptors, deletedDescriptors);
}
});
- return new HadoopXmlResourceParserResult(providers, descriptors);
+ return new HadoopXmlResourceParserResult(providers, descriptors, deletedDescriptors, deletedProviders);
+ }
+
+ private void parseProvider(String providerConfigurationName, String value, Map<String, ProviderConfiguration> providers, Set<String> deletedProviders) {
+ if (gatewayConfig.getReadOnlyOverrideProviderNames().contains(providerConfigurationName)) {
+ log.skipReadOnlyProvider(providerConfigurationName);
+ return;
+ }
+ final File providerConfigFile = resolveProviderConfiguration(providerConfigurationName);
+ try {
+ final ProviderConfiguration providerConfiguration = getProviderConfiguration(providers, providerConfigFile, providerConfigurationName);
+ providerConfiguration.setReadOnly(true);
+ if (isRemoved(value)) {
+ deletedProviders.add(providerConfigurationName);
+ } else {
+ providerConfiguration.saveOrUpdateProviders(parseProviderConfigurations(value, providerConfiguration));
+ providers.put(providerConfigurationName, providerConfiguration);
+ }
+ } catch (Exception e) {
+ log.failedToParseProviderConfiguration(providerConfigurationName, e.getMessage(), e);
+ }
+ }
+
+ private void parseDescriptor(String topologyName, String value, Set<SimpleDescriptor> descriptors, Set<String> deletedDescriptors) {
+ if (gatewayConfig.getReadOnlyOverrideTopologyNames().contains(topologyName)) {
+ log.skipReadOnlyDescriptor(topologyName);
+ return;
+ }
+ if (isRemoved(value)) {
+ deletedDescriptors.add(topologyName);
+ } else {
+ SimpleDescriptor descriptor = parseXmlDescriptor(topologyName, value);
+ if (descriptor != null) {
+ descriptors.add(descriptor);
+ }
+ }
+ }
+
+ private boolean isRemoved(String value) {
+ return value.trim().equalsIgnoreCase(REMOVE);
}
private ProviderConfiguration getProviderConfiguration(Map<String, ProviderConfiguration> providers, File providerConfigFile, String providerConfigName)
@@ -196,14 +222,14 @@ public class HadoopXmlResourceParser implements AdvancedServiceDiscoveryConfigCh
private ProviderConfiguration.Provider parseProvider(List<String> configurationPairs, String role, ProviderConfiguration providerConfiguration) {
final JSONProvider provider = new JSONProvider();
provider.setRole(role);
- getParamsForRole(role, providerConfiguration).forEach((key, value) -> provider.addParam(key, value)); //initializing parameters (if any)
+ getParamsForRole(role, providerConfiguration).forEach(provider::addParam); //initializing parameters (if any)
provider.setEnabled(true); //may be overwritten later, but defaulting to 'true'
final Set<String> roleConfigurations = configurationPairs.stream().filter(configurationPair -> configurationPair.trim().startsWith(role))
.map(configurationPair -> configurationPair.replace(role + ".", "").trim()).collect(Collectors.toSet());
for (String roleConfiguration : roleConfigurations) {
if (roleConfiguration.startsWith(CONFIG_NAME_PROVIDER_CONFIGS_PARAM_PREFIX)) {
String[] paramKeyValue = roleConfiguration.replace(CONFIG_NAME_PROVIDER_CONFIGS_PARAM_PREFIX, "").split("=", 2);
- if (CONFIG_NAME_PROVIDER_CONFIGS_PARAM_REMOVE.equals(paramKeyValue[0])) {
+ if (REMOVE.equals(paramKeyValue[0])) {
provider.removeParam(paramKeyValue[1]);
} else {
provider.addParam(paramKeyValue[0], paramKeyValue[1]);
diff --git a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceParserResult.java b/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceParserResult.java
index 378f73f75..1ab0e1897 100644
--- a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceParserResult.java
+++ b/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceParserResult.java
@@ -26,14 +26,19 @@ import org.apache.knox.gateway.topology.simple.SimpleDescriptor;
class HadoopXmlResourceParserResult {
final Map<String, ProviderConfiguration> providers;
final Set<SimpleDescriptor> descriptors;
+ private final Set<String> deletedDescriptors;
+ private final Set<String> deletedProviders;
HadoopXmlResourceParserResult() {
- this(Collections.emptyMap(), Collections.emptySet());
+ this(Collections.emptyMap(), Collections.emptySet(), Collections.emptySet(), Collections.emptySet());
}
- HadoopXmlResourceParserResult(Map<String, ProviderConfiguration> providers, Set<SimpleDescriptor> descriptors) {
+ HadoopXmlResourceParserResult(Map<String, ProviderConfiguration> providers, Set<SimpleDescriptor> descriptors,
+ Set<String> deletedDescriptors, Set<String> deletedProviders) {
this.providers = providers;
this.descriptors = descriptors;
+ this.deletedDescriptors = deletedDescriptors;
+ this.deletedProviders = deletedProviders;
}
public Map<String, ProviderConfiguration> getProviders() {
@@ -44,4 +49,11 @@ class HadoopXmlResourceParserResult {
return Collections.unmodifiableSet(descriptors);
}
+ public Set<String> getDeletedDescriptors() {
+ return deletedDescriptors;
+ }
+
+ public Set<String> getDeletedProviders() {
+ return deletedProviders;
+ }
}
diff --git a/gateway-topology-hadoop-xml/src/test/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceParserTest.java b/gateway-topology-hadoop-xml/src/test/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceParserTest.java
index e8ff1ceb5..dce2bff0c 100644
--- a/gateway-topology-hadoop-xml/src/test/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceParserTest.java
+++ b/gateway-topology-hadoop-xml/src/test/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceParserTest.java
@@ -26,7 +26,9 @@ import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -246,6 +248,14 @@ public class HadoopXmlResourceParserTest {
assertEquals("ClouderaManager", descriptor.getDiscoveryType());
}
+ @Test
+ public void testDelete() throws Exception {
+ String testConfigPath = this.getClass().getClassLoader().getResource("testDelete.xml").getPath();
+ HadoopXmlResourceParserResult result = hadoopXmlResourceParser.parse(testConfigPath);
+ assertEquals(new HashSet<>(Arrays.asList("topology1", "topology2")), result.getDeletedDescriptors());
+ assertEquals(new HashSet<>(Arrays.asList("admin", "knoxsso")), result.getDeletedProviders());
+ }
+
private void validateTopology1Descriptors(SimpleDescriptor descriptor) {
assertTrue(descriptor.isReadOnly());
assertEquals("topology1", descriptor.getName());
diff --git a/gateway-topology-hadoop-xml/src/test/resources/testDelete.xml b/gateway-topology-hadoop-xml/src/test/resources/testDelete.xml
new file mode 100644
index 000000000..609562757
--- /dev/null
+++ b/gateway-topology-hadoop-xml/src/test/resources/testDelete.xml
@@ -0,0 +1,66 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<configuration>
+ <property>
+ <name>topology1</name>
+ <value> remove </value>
+ </property>
+ <property>
+ <name>topology2</name>
+ <value>
+ remove
+ </value>
+ </property>
+ <property>
+ <name>topology3</name>
+ <value>
+ discoveryType=ClouderaManager#
+ discoveryAddress=http://host:123#
+ discoveryUser=user#
+ discoveryPasswordAlias=alias#
+ cluster=Cluster 1#
+ providerConfigRef=topology1-provider#
+ app:knoxauth:param1.name=param1.value#
+ app:admin-ui#
+ HIVE:url=http://localhost:456#
+ HIVE:version=1.0#
+ HIVE:httpclient.connectionTimeout=5m#
+ HIVE:httpclient.socketTimeout=100m
+ </value>
+ </property>
+ <property>
+ <name>providerConfigs:admin, knoxsso</name>
+ <value>remove</value>
+ </property>
+ <property>
+ <name>providerConfigs:prov2</name>
+ <value>
+ role=authentication#
+ authentication.name=ShiroProvider#
+ authentication.param.sessionTimeout=30#
+ authentication.param.main.ldapRealm=org.apache.hadoop.gateway.shirorealm.KnoxLdapRealm#
+ authentication.param.main.ldapContextFactory=org.apache.knox.gateway.shirorealm.KnoxLdapContextFactory#
+ authentication.param.main.ldapRealm.contextFactory=$ldapContextFactory#
+ authentication.param.main.ldapRealm.contextFactory.authenticationMechanism=simple#
+ authentication.param.main.ldapRealm.contextFactory.url=ldap://localhost:33389#
+ authentication.param.main.ldapRealm.contextFactory.systemUsername=uid=guest,ou=people,dc=hadoop,dc=apache,dc=org#
+ authentication.param.main.ldapRealm.contextFactory.systemPassword=${ALIAS=knoxLdapSystemPassword}#
+ authentication.param.main.ldapRealm.userDnTemplate=uid={0},ou=people,dc=hadoop,dc=apache,dc=org#
+ authentication.param.urls./**=authcBasic
+ </value>
+ </property>
+</configuration>
\ No newline at end of file