You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ja...@apache.org on 2017/10/09 18:34:56 UTC
[42/50] [abbrv] ambari git commit: AMBARI-21798. Oozie server crashes
post migration after regen kerberos keytabs and restart all services (rlevas)
AMBARI-21798. Oozie server crashes post migration after regen kerberos keytabs and restart all services (rlevas)
(cherry picked from commit 2eed29264cb06084bd4a4348e71421d419b22716)
Change-Id: Iec613a193d407e5f456011306b526758c37fb74e
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4b711613
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4b711613
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4b711613
Branch: refs/heads/2.5-maint
Commit: 4b711613c60000839822796a83891efcfc35f6d5
Parents: b41f06f
Author: Robert Levas <rl...@hortonworks.com>
Authored: Thu Aug 24 10:25:45 2017 -0400
Committer: rlevas@hortonworks.com <je...@hortonworks.com>
Committed: Thu Aug 24 14:51:19 2017 +0000
----------------------------------------------------------------------
.../server/upgrade/UpgradeCatalog252.java | 86 ++++++++++++++++----
.../server/upgrade/UpgradeCatalog252Test.java | 57 +++++++++++++
2 files changed, 128 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/4b711613/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java
index 0b422f8..718679b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java
@@ -44,9 +44,12 @@ import org.apache.ambari.server.state.ConfigHelper;
import org.apache.ambari.server.state.PropertyInfo;
import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.kerberos.AbstractKerberosDescriptorContainer;
+import org.apache.ambari.server.state.kerberos.KerberosComponentDescriptor;
import org.apache.ambari.server.state.kerberos.KerberosConfigurationDescriptor;
import org.apache.ambari.server.state.kerberos.KerberosDescriptor;
import org.apache.ambari.server.state.kerberos.KerberosDescriptorFactory;
+import org.apache.ambari.server.state.kerberos.KerberosServiceDescriptor;
+import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import com.google.common.collect.Sets;
@@ -352,15 +355,52 @@ public class UpgradeCatalog252 extends AbstractUpgradeCatalog {
final KerberosDescriptor kerberosDescriptor = new KerberosDescriptorFactory().createInstance(data);
if (kerberosDescriptor != null) {
+ boolean updated = false;
+
// Find and remove configuration specifications for <code>livy-conf/livy.superusers</code>
// in SPARK since this logic has been moved to the relevant stack/service advisors
- boolean updatedSpark = removeConfigurationSpecification(kerberosDescriptor.getService("SPARK"), "livy-conf", "livy.superusers");
+ if(removeConfigurationSpecifications(kerberosDescriptor.getService("SPARK"),
+ Collections.<String, Collection<String>>singletonMap("livy-conf", Collections.singleton("livy.superusers")))) {
+ updated = true;
+ }
// Find and remove configuration specifications for <code>livy-conf2/livy.superusers</code>
// in SPARK2 since this logic has been moved to the relevant stack/service advisors
- boolean updatedSpark2 = removeConfigurationSpecification(kerberosDescriptor.getService("SPARK2"), "livy2-conf", "livy.superusers");
+ if(removeConfigurationSpecifications(kerberosDescriptor.getService("SPARK2"),
+ Collections.<String, Collection<String>>singletonMap("livy2-conf", Collections.singleton("livy.superusers")))) {
+ updated = true;
+ }
- if (updatedSpark || updatedSpark2) {
+ // Find and remove configuration specifications for the following configurations in KNOX/KNOX_GATEWAY
+ // since they are invalid due to static "knox" embedded in the property name:
+ // * oozie-site/oozie.service.ProxyUserService.proxyuser.knox.groups
+ // * oozie-site/oozie.service.ProxyUserService.proxyuser.knox.hosts
+ // * webhcat-site/webhcat.proxyuser.knox.groups
+ // * webhcat-site/webhcat.proxyuser.knox.hosts
+ // * core-site/hadoop.proxyuser.knox.groups
+ // * core-site/hadoop.proxyuser.knox.hosts
+ // * falcon-runtime.properties/*.falcon.service.ProxyUserService.proxyuser.knox.groups
+ // * falcon-runtime.properties/*.falcon.service.ProxyUserService.proxyuser.knox.hosts
+ KerberosServiceDescriptor knoxKerberosDescriptor = kerberosDescriptor.getService("KNOX");
+ if(knoxKerberosDescriptor != null) {
+ KerberosComponentDescriptor knoxGatewayKerberosDescriptor = knoxKerberosDescriptor.getComponent("KNOX_GATEWAY");
+ if (knoxGatewayKerberosDescriptor != null) {
+ Map<String, Collection<String>> configsToRemove = new HashMap<>();
+ configsToRemove.put("oozie-site",
+ Arrays.asList("oozie.service.ProxyUserService.proxyuser.knox.groups", "oozie.service.ProxyUserService.proxyuser.knox.hosts"));
+ configsToRemove.put("webhcat-site",
+ Arrays.asList("webhcat.proxyuser.knox.groups", "webhcat.proxyuser.knox.hosts"));
+ configsToRemove.put("core-site",
+ Arrays.asList("hadoop.proxyuser.knox.groups", "hadoop.proxyuser.knox.hosts"));
+ configsToRemove.put("falcon-runtime.properties",
+ Arrays.asList("*.falcon.service.ProxyUserService.proxyuser.knox.groups", "*.falcon.service.ProxyUserService.proxyuser.knox.hosts"));
+ if (removeConfigurationSpecifications(knoxGatewayKerberosDescriptor, configsToRemove)) {
+ updated = true;
+ }
+ }
+ }
+
+ if (updated) {
artifactEntity.setArtifactData(kerberosDescriptor.toMap());
artifactDAO.merge(artifactEntity);
}
@@ -470,24 +510,40 @@ public class UpgradeCatalog252 extends AbstractUpgradeCatalog {
}
/**
- * Given an {@link AbstractKerberosDescriptorContainer}, attempts to remove the specified property
- * (<code>configType/propertyName</code> from it.
+ * Given an {@link AbstractKerberosDescriptorContainer}, attempts to remove the specified
+ * configurations (<code>configType/propertyName</code> from it.
*
* @param kerberosDescriptorContainer the container to update
- * @param configType the configuration type
- * @param propertyName the property name
+ * @param configurations a map of configuration types to sets of property names.
* @return true if changes where made to the container; false otherwise
*/
- private boolean removeConfigurationSpecification(AbstractKerberosDescriptorContainer kerberosDescriptorContainer, String configType, String propertyName) {
+ private boolean removeConfigurationSpecifications(AbstractKerberosDescriptorContainer kerberosDescriptorContainer, Map<String, Collection<String>> configurations) {
boolean updated = false;
if (kerberosDescriptorContainer != null) {
- KerberosConfigurationDescriptor configurationDescriptor = kerberosDescriptorContainer.getConfiguration(configType);
- if (configurationDescriptor != null) {
- Map<String, String> properties = configurationDescriptor.getProperties();
- if ((properties != null) && properties.containsKey(propertyName)) {
- properties.remove(propertyName);
- LOG.info("Removed {}/{} from the descriptor named {}", configType, propertyName, kerberosDescriptorContainer.getName());
- updated = true;
+ if (!MapUtils.isEmpty(configurations)) {
+ for (Map.Entry<String, Collection<String>> entry : configurations.entrySet()) {
+ String configType = entry.getKey();
+
+ for (String propertyName : entry.getValue()) {
+ Map<String, KerberosConfigurationDescriptor> configurationDescriptors = kerberosDescriptorContainer.getConfigurations(false);
+ KerberosConfigurationDescriptor configurationDescriptor = (configurationDescriptors == null)
+ ? null
+ : configurationDescriptors.get(configType);
+ if (configurationDescriptor != null) {
+ Map<String, String> properties = configurationDescriptor.getProperties();
+ if ((properties != null) && properties.containsKey(propertyName)) {
+ properties.remove(propertyName);
+ LOG.info("Removed {}/{} from the descriptor named {}", configType, propertyName, kerberosDescriptorContainer.getName());
+ updated = true;
+
+ // If there are no more properties in the configurationDescriptor, remove it from the container.
+ if(properties.isEmpty()) {
+ configurationDescriptors.remove(configType);
+ kerberosDescriptorContainer.setConfigurations(configurationDescriptors);
+ }
+ }
+ }
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4b711613/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog252Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog252Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog252Test.java
index 72956eb..e6dbb7c 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog252Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog252Test.java
@@ -54,6 +54,7 @@ import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.Config;
import org.apache.ambari.server.state.Service;
+import org.apache.ambari.server.state.kerberos.KerberosComponentDescriptor;
import org.apache.ambari.server.state.kerberos.KerberosDescriptor;
import org.apache.ambari.server.state.kerberos.KerberosDescriptorFactory;
import org.apache.ambari.server.state.stack.OsFamily;
@@ -289,6 +290,48 @@ public class UpgradeCatalog252Test {
" ]" +
" }," +
" {" +
+ " \"name\": \"KNOX\"," +
+ " \"components\": [" +
+ " {" +
+ " \"name\": \"KNOX_GATEWAY\"," +
+ " \"configurations\": [" +
+ " {" +
+ " \"core-site\": {" +
+ " \"property1\": \"true\"," +
+ " \"property2\": \"true\"," +
+ " \"hadoop.proxyuser.knox.groups\": \"somevalue\"," +
+ " \"hadoop.proxyuser.knox.hosts\": \"somevalue\"" +
+ " }" +
+ " }," +
+ " {" +
+ " \"webhcat-site\": {" +
+ " \"webhcat.proxyuser.knox.groups\": \"somevalue\"," +
+ " \"webhcat.proxyuser.knox.hosts\": \"somevalue\"" +
+ " }" +
+ " }," +
+ " {" +
+ " \"oozie-site\": {" +
+ " \"oozie.service.ProxyUserService.proxyuser.knox.groups\": \"somevalue\"," +
+ " \"oozie.service.ProxyUserService.proxyuser.knox.hosts\": \"somevalue\"" +
+ " }" +
+ " }," +
+ " {" +
+ " \"falcon-runtime.properties\": {" +
+ " \"*.falcon.service.ProxyUserService.proxyuser.knox.groups\": \"somevalue\"," +
+ " \"*.falcon.service.ProxyUserService.proxyuser.knox.hosts\": \"somevalue\"" +
+ " }" +
+ " }," +
+ " {" +
+ " \"some-env\": {" +
+ " \"groups\": \"${hadoop-env/proxyuser_group}\"," +
+ " \"hosts\": \"${clusterHostInfo/existing_service_master_hosts}\"" +
+ " }" +
+ " }" +
+ " ]" +
+ " }" +
+ " ]" +
+ " }," +
+ " {" +
" \"name\": \"NOT_SPARK\"," +
" \"configurations\": [" +
" {" +
@@ -360,5 +403,19 @@ public class UpgradeCatalog252Test {
Assert.assertNotNull(result.getService("NOT_SPARK").getConfiguration("not-livy-conf"));
Assert.assertNotNull(result.getService("NOT_SPARK").getConfiguration("not-livy-conf").getProperties());
Assert.assertTrue(result.getService("NOT_SPARK").getConfiguration("not-livy-conf").getProperties().containsKey("livy.superusers"));
+
+ Assert.assertNotNull(result.getService("KNOX"));
+
+ KerberosComponentDescriptor knoxGateway = result.getService("KNOX").getComponent("KNOX_GATEWAY");
+ Assert.assertNotNull(knoxGateway);
+ Assert.assertNotNull(knoxGateway.getConfiguration("core-site"));
+ Assert.assertNotNull(knoxGateway.getConfiguration("core-site").getProperties());
+ Assert.assertTrue(knoxGateway.getConfiguration("core-site").getProperties().containsKey("property1"));
+ Assert.assertFalse(knoxGateway.getConfiguration("core-site").getProperties().containsKey("hadoop.proxyuser.knox.groups"));
+ Assert.assertFalse(knoxGateway.getConfiguration("core-site").getProperties().containsKey("hadoop.proxyuser.knox.hosts"));
+ Assert.assertNull(knoxGateway.getConfiguration("oozie-site"));
+ Assert.assertNull(knoxGateway.getConfiguration("webhcat-site"));
+ Assert.assertNull(knoxGateway.getConfiguration("falcon-runtime.properties"));
+ Assert.assertNotNull(knoxGateway.getConfiguration("some-env"));
}
}