You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by dm...@apache.org on 2017/01/17 10:07:16 UTC
[2/4] ambari git commit: AMBARI-19561. After ambari only upgrade
Property:"yarn.nodemanager.linux-container-executor.cgroups.mount-path"
become required (dgrinenko via dlysnichenko)
AMBARI-19561. After ambari only upgrade Property:"yarn.nodemanager.linux-container-executor.cgroups.mount-path" become required (dgrinenko via dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d9204601
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d9204601
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d9204601
Branch: refs/heads/branch-2.5
Commit: d9204601f9231caeea67f4e0aa887888310a2ddb
Parents: 8ee6196
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Tue Jan 17 12:04:06 2017 +0200
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Tue Jan 17 12:05:59 2017 +0200
----------------------------------------------------------------------
.../server/upgrade/UpgradeCatalog250.java | 33 ++++++
.../YARN/2.1.0.2.0/kerberos.json | 1 -
.../stacks/HDP/2.2/services/YARN/kerberos.json | 1 -
.../HDP/2.3.ECS/services/YARN/kerberos.json | 1 -
.../stacks/HDP/2.3/services/YARN/kerberos.json | 1 -
.../stacks/HDP/2.5/services/YARN/kerberos.json | 1 -
.../stacks/PERF/1.0/services/YARN/kerberos.json | 1 -
.../server/upgrade/UpgradeCatalog250Test.java | 106 +++++++++++++++++++
.../2.2/configs/pig-service-check-secure.json | 3 +-
.../test_kerberos_descriptor_2_1_3.json | 1 -
.../data/stacks/HDP-2.1/service_components.json | 1 -
11 files changed, 140 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/d9204601/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
index cb509cd..c839b18 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
@@ -64,6 +64,10 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
private static final String HBASE_ROOTDIR = "hbase.rootdir";
private static final String HADOOP_ENV = "hadoop-env";
private static final String KAFKA_BROKER = "kafka-broker";
+ private static final String YARN_SITE_CONFIG = "yarn-site";
+ private static final String YARN_ENV_CONFIG = "yarn-env";
+ private static final String YARN_LCE_CGROUPS_MOUNT_PATH = "yarn.nodemanager.linux-container-executor.cgroups.mount-path";
+ private static final String YARN_CGROUPS_ENABLED = "yarn_cgroups_enabled";
private static final String KAFKA_TIMELINE_METRICS_HOST = "kafka.timeline.metrics.host";
public static final String COMPONENT_TABLE = "servicecomponentdesiredstate";
@@ -161,6 +165,7 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
updateAtlasConfigs();
updateLogSearchConfigs();
updateAmbariInfraConfigs();
+ updateYarnSite();
addManageServiceAutoStartPermissions();
}
@@ -179,6 +184,34 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
dbAccessor.addUniqueConstraint(GROUPS_TABLE, "UNQ_groups_0", "group_name", "group_type");
}
+ /**
+ * Updates {@code yarn-site} in the following ways:
+ *
+ * Remove {@code YARN_LCE_CGROUPS_MOUNT_PATH} if {@code YARN_CGROUPS_ENABLED} is {@code false} and
+ * {@code YARN_LCE_CGROUPS_MOUNT_PATH} is empty string
+ *
+ * @throws AmbariException
+ */
+ protected void updateYarnSite() throws AmbariException {
+ AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
+ Clusters clusters = ambariManagementController.getClusters();
+ Map<String, Cluster> clusterMap = getCheckedClusterMap(clusters);
+ for (final Cluster cluster : clusterMap.values()) {
+ Config yarnEnvConfig = cluster.getDesiredConfigByType(YARN_ENV_CONFIG);
+ Config yarnSiteConfig = cluster.getDesiredConfigByType(YARN_SITE_CONFIG);
+ if (yarnEnvConfig != null && yarnSiteConfig != null) {
+ String cgroupEnabled = yarnEnvConfig.getProperties().get(YARN_CGROUPS_ENABLED);
+ String mountPath = yarnSiteConfig.getProperties().get(YARN_LCE_CGROUPS_MOUNT_PATH);
+ if (StringUtils.isEmpty(mountPath) && cgroupEnabled != null
+ && cgroupEnabled.trim().equalsIgnoreCase("false")){
+ removeConfigurationPropertiesFromCluster(cluster, YARN_SITE_CONFIG, new HashSet<String>(){{
+ add(YARN_LCE_CGROUPS_MOUNT_PATH);
+ }});
+ }
+ }
+ }
+ }
+
protected void updateAMSConfigs() throws AmbariException {
AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
Clusters clusters = ambariManagementController.getClusters();
http://git-wip-us.apache.org/repos/asf/ambari/blob/d9204601/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/kerberos.json b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/kerberos.json
index c307800..6b61c13 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/kerberos.json
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/kerberos.json
@@ -31,7 +31,6 @@
"yarn.resourcemanager.proxyusers.*.hosts": "",
"yarn.resourcemanager.proxyusers.*.users": "",
"yarn.resourcemanager.proxy-user-privileges.enabled": "true",
- "yarn.nodemanager.linux-container-executor.cgroups.mount-path": "",
"yarn.resourcemanager.zk-acl" : "sasl:rm:rwcda",
"hadoop.registry.secure" : "true",
"hadoop.registry.system.accounts" : "sasl:yarn,sasl:mapred,sasl:hadoop,sasl:hdfs,sasl:rm"
http://git-wip-us.apache.org/repos/asf/ambari/blob/d9204601/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/kerberos.json
index 3a183cc..ad30b76 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/kerberos.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/kerberos.json
@@ -31,7 +31,6 @@
"yarn.resourcemanager.proxyusers.*.hosts": "",
"yarn.resourcemanager.proxyusers.*.users": "",
"yarn.resourcemanager.proxy-user-privileges.enabled": "true",
- "yarn.nodemanager.linux-container-executor.cgroups.mount-path": "",
"yarn.resourcemanager.zk-state-store.parent-path": "/rmstore-secure",
"yarn.resourcemanager.zk-acl" : "sasl:rm:rwcda",
"hadoop.registry.secure" : "true",
http://git-wip-us.apache.org/repos/asf/ambari/blob/d9204601/ambari-server/src/main/resources/stacks/HDP/2.3.ECS/services/YARN/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3.ECS/services/YARN/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.3.ECS/services/YARN/kerberos.json
index e11ce84..7977941 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3.ECS/services/YARN/kerberos.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3.ECS/services/YARN/kerberos.json
@@ -34,7 +34,6 @@
"yarn.resourcemanager.proxyusers.*.hosts": "",
"yarn.resourcemanager.proxyusers.*.users": "",
"yarn.resourcemanager.proxy-user-privileges.enabled": "true",
- "yarn.nodemanager.linux-container-executor.cgroups.mount-path": "",
"yarn.resourcemanager.zk-acl" : "sasl:rm:rwcda",
"hadoop.registry.secure" : "true",
"hadoop.registry.system.accounts" : "sasl:yarn,sasl:mapred,sasl:hadoop,sasl:hdfs,sasl:rm"
http://git-wip-us.apache.org/repos/asf/ambari/blob/d9204601/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/kerberos.json
index 1a6cf5b..73addb1 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/kerberos.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/kerberos.json
@@ -32,7 +32,6 @@
"yarn.resourcemanager.proxyusers.*.hosts": "",
"yarn.resourcemanager.proxyusers.*.users": "",
"yarn.resourcemanager.proxy-user-privileges.enabled": "true",
- "yarn.nodemanager.linux-container-executor.cgroups.mount-path": "",
"yarn.resourcemanager.zk-acl" : "sasl:rm:rwcda",
"hadoop.registry.secure" : "true",
"hadoop.registry.system.accounts" : "sasl:yarn,sasl:mapred,sasl:hadoop,sasl:hdfs,sasl:rm"
http://git-wip-us.apache.org/repos/asf/ambari/blob/d9204601/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json
index af920f1..d334887 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json
@@ -32,7 +32,6 @@
"yarn.resourcemanager.proxyusers.*.hosts": "",
"yarn.resourcemanager.proxyusers.*.users": "",
"yarn.resourcemanager.proxy-user-privileges.enabled": "true",
- "yarn.nodemanager.linux-container-executor.cgroups.mount-path": "",
"yarn.resourcemanager.zk-acl" : "sasl:rm:rwcda",
"hadoop.registry.secure" : "true",
"hadoop.registry.system.accounts" : "sasl:yarn,sasl:mapred,sasl:hadoop,sasl:hdfs,sasl:rm"
http://git-wip-us.apache.org/repos/asf/ambari/blob/d9204601/ambari-server/src/main/resources/stacks/PERF/1.0/services/YARN/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/PERF/1.0/services/YARN/kerberos.json b/ambari-server/src/main/resources/stacks/PERF/1.0/services/YARN/kerberos.json
index f33c07d..7e74237 100644
--- a/ambari-server/src/main/resources/stacks/PERF/1.0/services/YARN/kerberos.json
+++ b/ambari-server/src/main/resources/stacks/PERF/1.0/services/YARN/kerberos.json
@@ -32,7 +32,6 @@
"yarn.resourcemanager.proxyusers.*.hosts": "",
"yarn.resourcemanager.proxyusers.*.users": "",
"yarn.resourcemanager.proxy-user-privileges.enabled": "true",
- "yarn.nodemanager.linux-container-executor.cgroups.mount-path": "",
"yarn.resourcemanager.zk-state-store.parent-path": "/rmstore-secure"
}
},
http://git-wip-us.apache.org/repos/asf/ambari/blob/d9204601/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
index 52bc02f..44d52cf 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
@@ -27,6 +27,7 @@ import com.google.inject.Injector;
import com.google.inject.Module;
import com.google.inject.Provider;
import junit.framework.Assert;
+import junit.framework.AssertionFailedError;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.actionmanager.ActionManager;
@@ -67,13 +68,17 @@ import java.sql.Statement;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
+
import static org.easymock.EasyMock.anyObject;
import static org.easymock.EasyMock.anyString;
import static org.easymock.EasyMock.capture;
import static org.easymock.EasyMock.createMockBuilder;
+import static org.easymock.EasyMock.createNiceMock;
import static org.easymock.EasyMock.eq;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.expectLastCall;
@@ -339,6 +344,107 @@ public class UpgradeCatalog250Test {
}
@Test
+ public void testUpdateYarnSite() throws Exception{
+ EasyMockSupport easyMockSupport = new EasyMockSupport();
+
+ final String propertyToRemove = "yarn.nodemanager.linux-container-executor.cgroups.mount-path";
+ final AmbariManagementController ambariManagementController = createNiceMock(AmbariManagementController.class);
+ Config mockYarnEnv = easyMockSupport.createNiceMock(Config.class);
+ Config mockYarnSite = easyMockSupport.createNiceMock(Config.class);
+
+ HashMap<String, String> yarnEnv = new HashMap<String, String>(){{
+ put("yarn_cgroups_enabled", "false");
+ }};
+
+ HashMap<String, String> yarnSite = new HashMap<String, String>() {{
+ put(propertyToRemove, "");
+ }};
+
+ reset(clusters, cluster, injector);
+
+ expect(injector.getInstance(AmbariManagementController.class)).andReturn(ambariManagementController).atLeastOnce();
+ expect(ambariManagementController.getClusters()).andReturn(clusters).atLeastOnce();
+ expect(clusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{
+ put("normal", cluster);
+ }}).once();
+ expect(cluster.getDesiredConfigByType("yarn-env")).andReturn(mockYarnEnv).atLeastOnce();
+ expect(mockYarnEnv.getProperties()).andReturn(yarnEnv).anyTimes();
+ expect(cluster.getDesiredConfigByType("yarn-site")).andReturn(mockYarnSite).atLeastOnce();
+ expect(mockYarnSite.getProperties()).andReturn(yarnSite).anyTimes();
+
+ replay(clusters, cluster, injector, ambariManagementController, mockYarnEnv, mockYarnSite);
+
+ UpgradeCatalog250 upgradeCatalog250 = createMockBuilder(UpgradeCatalog250.class)
+ .addMockedMethod("removeConfigurationPropertiesFromCluster")
+ .withConstructor(injector)
+ .createNiceMock();
+
+ Capture<HashSet<String>> removeConfigName = EasyMock.newCapture();
+
+ upgradeCatalog250.removeConfigurationPropertiesFromCluster(anyObject(Cluster.class), eq("yarn-site"), capture(removeConfigName));
+ EasyMock.expectLastCall();
+
+ replay(upgradeCatalog250);
+
+ upgradeCatalog250.updateYarnSite();
+
+ easyMockSupport.verifyAll();
+
+ Set<String> updatedProperties = removeConfigName.getValue();
+ assertTrue(updatedProperties.contains(propertyToRemove));
+
+ reset(injector);
+ }
+
+ @Test
+ public void testUpdateYarnSiteWithEnabledCGroups() throws Exception{
+ EasyMockSupport easyMockSupport = new EasyMockSupport();
+
+ final String propertyToRemove = "yarn.nodemanager.linux-container-executor.cgroups.mount-path";
+ final AmbariManagementController ambariManagementController = createNiceMock(AmbariManagementController.class);
+ Config mockYarnEnv = easyMockSupport.createNiceMock(Config.class);
+ Config mockYarnSite = easyMockSupport.createNiceMock(Config.class);
+
+ HashMap<String, String> yarnEnv = new HashMap<String, String>(){{
+ put("yarn_cgroups_enabled", "true");
+ }};
+
+ HashMap<String, String> yarnSite = new HashMap<String, String>() {{
+ put(propertyToRemove, "");
+ }};
+
+ reset(clusters, cluster, injector);
+
+ expect(injector.getInstance(AmbariManagementController.class)).andReturn(ambariManagementController).atLeastOnce();
+ expect(ambariManagementController.getClusters()).andReturn(clusters).atLeastOnce();
+ expect(clusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{
+ put("normal", cluster);
+ }}).once();
+ expect(cluster.getDesiredConfigByType("yarn-env")).andReturn(mockYarnEnv).atLeastOnce();
+ expect(mockYarnEnv.getProperties()).andReturn(yarnEnv).anyTimes();
+ expect(cluster.getDesiredConfigByType("yarn-site")).andReturn(mockYarnSite).atLeastOnce();
+ expect(mockYarnSite.getProperties()).andReturn(yarnSite).anyTimes();
+
+ replay(clusters, cluster, injector, ambariManagementController, mockYarnEnv, mockYarnSite);
+
+ UpgradeCatalog250 upgradeCatalog250 = createMockBuilder(UpgradeCatalog250.class)
+ .addMockedMethod("removeConfigurationPropertiesFromCluster")
+ .withConstructor(injector)
+ .createNiceMock();
+
+ Capture<HashSet<String>> removeConfigName = EasyMock.newCapture();
+
+ upgradeCatalog250.removeConfigurationPropertiesFromCluster(anyObject(Cluster.class), eq("yarn-site"), capture(removeConfigName));
+ EasyMock.expectLastCall().andThrow(new AssertionFailedError()).anyTimes();
+
+ replay(upgradeCatalog250);
+
+ upgradeCatalog250.updateYarnSite();
+
+ reset(injector);
+ }
+
+ @Test
public void testAmsEnvUpdateConfigs() throws Exception {
Map<String, String> oldPropertiesAmsEnv = new HashMap<String, String>() {
http://git-wip-us.apache.org/repos/asf/ambari/blob/d9204601/ambari-server/src/test/python/stacks/2.2/configs/pig-service-check-secure.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/configs/pig-service-check-secure.json b/ambari-server/src/test/python/stacks/2.2/configs/pig-service-check-secure.json
index 0d0c6f5..f14eb52 100644
--- a/ambari-server/src/test/python/stacks/2.2/configs/pig-service-check-secure.json
+++ b/ambari-server/src/test/python/stacks/2.2/configs/pig-service-check-secure.json
@@ -470,8 +470,7 @@
"yarn.resourcemanager.recovery.enabled": "true",
"yarn.timeline-service.http-authentication.cookie.domain": "",
"yarn.resourcemanager.zk-retry-interval-ms": "1000",
- "yarn.nodemanager.linux-container-executor.cgroups.mount-path": "",
- "yarn.admin.acl": "",
+ "yarn.admin.acl": "",
"yarn.node-labels.fs-store.root-dir": "/system/yarn/node-labels",
"yarn.client.nodemanager-connect.retry-interval-ms": "10000",
"yarn.resourcemanager.admin.address": "c6402.ambari.apache.org:8141",
http://git-wip-us.apache.org/repos/asf/ambari/blob/d9204601/ambari-server/src/test/resources/kerberos/test_kerberos_descriptor_2_1_3.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/kerberos/test_kerberos_descriptor_2_1_3.json b/ambari-server/src/test/resources/kerberos/test_kerberos_descriptor_2_1_3.json
index fbe7857..5688d43 100644
--- a/ambari-server/src/test/resources/kerberos/test_kerberos_descriptor_2_1_3.json
+++ b/ambari-server/src/test/resources/kerberos/test_kerberos_descriptor_2_1_3.json
@@ -802,7 +802,6 @@
"yarn.timeline-service.http-authentication.kerberos.name.rules": "",
"yarn.timeline-service.http-authentication.cookie.path": "",
"yarn.timeline-service.http-authentication.type": "kerberos",
- "yarn.nodemanager.linux-container-executor.cgroups.mount-path": "",
"yarn.resourcemanager.proxy-user-privileges.enabled": "true",
"yarn.acl.enable": "true",
"yarn.timeline-service.http-authentication.signer.secret.provider": "",
http://git-wip-us.apache.org/repos/asf/ambari/blob/d9204601/ambari-web/app/assets/data/stacks/HDP-2.1/service_components.json
----------------------------------------------------------------------
diff --git a/ambari-web/app/assets/data/stacks/HDP-2.1/service_components.json b/ambari-web/app/assets/data/stacks/HDP-2.1/service_components.json
index d853f24..147c1c0 100644
--- a/ambari-web/app/assets/data/stacks/HDP-2.1/service_components.json
+++ b/ambari-web/app/assets/data/stacks/HDP-2.1/service_components.json
@@ -2898,7 +2898,6 @@
"yarn.timeline-service.http-authentication.kerberos.name.rules" : "",
"yarn.timeline-service.http-authentication.cookie.path" : "",
"yarn.timeline-service.http-authentication.type" : "kerberos",
- "yarn.nodemanager.linux-container-executor.cgroups.mount-path" : "",
"yarn.resourcemanager.proxy-user-privileges.enabled" : "true",
"yarn.acl.enable" : "true",
"yarn.timeline-service.http-authentication.signer.secret.provider" : "",