You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2016/07/26 18:03:42 UTC
ambari git commit: AMBARI-17894 - Adding Services After A
Stack/Ambari Upgrade Shows Empty Required Values (jonathanhurley)
Repository: ambari
Updated Branches:
refs/heads/trunk 5ebb04a2e -> 5261c76c6
AMBARI-17894 - Adding Services After A Stack/Ambari Upgrade Shows Empty Required Values (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5261c76c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5261c76c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5261c76c
Branch: refs/heads/trunk
Commit: 5261c76c62ec06af77d04bc5e0a9f128d3be12b2
Parents: 5ebb04a
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Mon Jul 25 17:03:07 2016 -0400
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Tue Jul 26 14:03:31 2016 -0400
----------------------------------------------------------------------
.../internal/UpgradeResourceProvider.java | 55 +++-
.../HIVE/configuration/hiveserver2-site.xml | 6 +-
.../StackUpgradeConfigurationMergeTest.java | 298 +++++++++++++++++++
3 files changed, 344 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/5261c76c/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
index 55e9dbd..255cbbb 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
@@ -1178,6 +1178,15 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
continue;
}
+ // get current stack default configurations on install
+ Map<String, String> configurationTypeDefaultConfigurations = oldStackDefaultConfigurationsByType.get(
+ configurationType);
+
+ // NPE sanity for current stack defaults
+ if (null == configurationTypeDefaultConfigurations) {
+ configurationTypeDefaultConfigurations = Collections.emptyMap();
+ }
+
// get the existing configurations
Map<String, String> existingConfigurations = currentClusterConfig.getProperties();
@@ -1214,22 +1223,19 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
if (newDefaultConfigurations.containsKey(existingConfigurationKey)) {
String newDefaultConfigurationValue = newDefaultConfigurations.get(
existingConfigurationKey);
+
if (!StringUtils.equals(existingConfigurationValue, newDefaultConfigurationValue)) {
// the new default is different from the existing cluster value;
// only override the default value if the existing value differs
// from the original stack
- Map<String, String> configurationTypeDefaultConfigurations = oldStackDefaultConfigurationsByType.get(
- configurationType);
- if (null != configurationTypeDefaultConfigurations) {
- String oldDefaultValue = configurationTypeDefaultConfigurations.get(
- existingConfigurationKey);
- if (!StringUtils.equals(existingConfigurationValue, oldDefaultValue)) {
- // at this point, we've determined that there is a difference
- // between default values between stacks, but the value was
- // also customized, so keep the customized value
- newDefaultConfigurations.put(existingConfigurationKey,
- existingConfigurationValue);
- }
+ String oldDefaultValue = configurationTypeDefaultConfigurations.get(
+ existingConfigurationKey);
+
+ if (!StringUtils.equals(existingConfigurationValue, oldDefaultValue)) {
+ // at this point, we've determined that there is a difference
+ // between default values between stacks, but the value was
+ // also customized, so keep the customized value
+ newDefaultConfigurations.put(existingConfigurationKey, existingConfigurationValue);
}
}
} else {
@@ -1237,6 +1243,31 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
newDefaultConfigurations.put(existingConfigurationKey, existingConfigurationValue);
}
}
+
+ /*
+ for every new configuration which does not exist in the existing
+ configurations, see if it was present in the current stack
+
+ stack 2.x has foo-site/property (on-ambari-upgrade is false)
+ stack 2.y has foo-site/property
+ the current cluster (on 2.x) does not have it
+
+ In this case, we should NOT add it back as clearly stack advisor has removed it
+ */
+ Iterator<Map.Entry<String, String>> newDefaultConfigurationsIterator = newDefaultConfigurations.entrySet().iterator();
+ while( newDefaultConfigurationsIterator.hasNext() ){
+ Map.Entry<String, String> newConfigurationEntry = newDefaultConfigurationsIterator.next();
+ String newConfigurationPropertyName = newConfigurationEntry.getKey();
+ if (configurationTypeDefaultConfigurations.containsKey(newConfigurationPropertyName)
+ && !existingConfigurations.containsKey(newConfigurationPropertyName)) {
+ LOG.info(
+ "The property {}/{} exists in both {} and {} but is not part of the current set of configurations and will therefore not be included in the configuration merge",
+ configurationType, newConfigurationPropertyName, currentStackId, targetStackId);
+
+ // remove the property so it doesn't get merged in
+ newDefaultConfigurationsIterator.remove();
+ }
+ }
}
} else {
// downgrade
http://git-wip-us.apache.org/repos/asf/ambari/blob/5261c76c/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hiveserver2-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hiveserver2-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hiveserver2-site.xml
index ad37114..448373f 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hiveserver2-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hiveserver2-site.xml
@@ -27,7 +27,7 @@ limitations under the License.
<name>hive_security_authorization</name>
</property>
</depends-on>
- <on-ambari-upgrade add="true"/>
+ <on-ambari-upgrade add="false"/>
</property>
<property>
<name>hive.security.authorization.manager</name>
@@ -40,7 +40,7 @@ limitations under the License.
<name>hive_security_authorization</name>
</property>
</depends-on>
- <on-ambari-upgrade add="true"/>
+ <on-ambari-upgrade add="false"/>
</property>
<property>
<name>hive.security.authorization.enabled</name>
@@ -79,6 +79,6 @@ limitations under the License.
<name>hive_security_authorization</name>
</property>
</depends-on>
- <on-ambari-upgrade add="true"/>
+ <on-ambari-upgrade add="false"/>
</property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/5261c76c/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackUpgradeConfigurationMergeTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackUpgradeConfigurationMergeTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackUpgradeConfigurationMergeTest.java
new file mode 100644
index 0000000..4f2580a
--- /dev/null
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackUpgradeConfigurationMergeTest.java
@@ -0,0 +1,298 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.controller.internal;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.persistence.EntityManager;
+
+import org.apache.ambari.server.actionmanager.ActionManager;
+import org.apache.ambari.server.actionmanager.RequestFactory;
+import org.apache.ambari.server.actionmanager.StageFactory;
+import org.apache.ambari.server.api.services.AmbariMetaInfo;
+import org.apache.ambari.server.controller.AbstractRootServiceResponseFactory;
+import org.apache.ambari.server.controller.AmbariManagementController;
+import org.apache.ambari.server.controller.KerberosHelper;
+import org.apache.ambari.server.controller.spi.ClusterController;
+import org.apache.ambari.server.orm.DBAccessor;
+import org.apache.ambari.server.orm.dao.HostRoleCommandDAO;
+import org.apache.ambari.server.orm.dao.RepositoryVersionDAO;
+import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
+import org.apache.ambari.server.orm.entities.StackEntity;
+import org.apache.ambari.server.scheduler.ExecutionScheduler;
+import org.apache.ambari.server.security.authorization.Users;
+import org.apache.ambari.server.stack.StackManagerFactory;
+import org.apache.ambari.server.stageplanner.RoleGraphFactory;
+import org.apache.ambari.server.stageplanner.RoleGraphFactoryImpl;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Config;
+import org.apache.ambari.server.state.ConfigFactory;
+import org.apache.ambari.server.state.ConfigHelper;
+import org.apache.ambari.server.state.DesiredConfig;
+import org.apache.ambari.server.state.ServiceComponentFactory;
+import org.apache.ambari.server.state.ServiceComponentHostFactory;
+import org.apache.ambari.server.state.ServiceFactory;
+import org.apache.ambari.server.state.ServiceInfo;
+import org.apache.ambari.server.state.StackId;
+import org.apache.ambari.server.state.configgroup.ConfigGroupFactory;
+import org.apache.ambari.server.state.scheduler.RequestExecutionFactory;
+import org.apache.ambari.server.state.stack.OsFamily;
+import org.apache.ambari.server.state.stack.UpgradePack;
+import org.apache.ambari.server.state.stack.upgrade.Direction;
+import org.apache.ambari.server.state.stack.upgrade.Grouping;
+import org.easymock.Capture;
+import org.easymock.EasyMock;
+import org.easymock.EasyMockSupport;
+import org.eclipse.jetty.server.SessionManager;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.springframework.security.crypto.password.PasswordEncoder;
+
+import com.google.inject.Binder;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.Module;
+
+import junit.framework.Assert;
+
+/**
+ * Tests that
+ * {@link UpgradeResourceProvider#applyStackAndProcessConfigurations(String, Cluster, String, Direction, UpgradePack, String)}
+ * works correctly.
+ */
+public class StackUpgradeConfigurationMergeTest extends EasyMockSupport {
+
+ private static final StackId s_currentStackId = new StackId("HDP-2.4");
+ private static final StackId s_targetStackId = new StackId("HDP-2.5");
+
+ private Injector m_injector;
+ private Clusters m_clustersMock;
+ private AmbariMetaInfo m_ambariMetaInfoMock;
+
+ /**
+ * @throws Exception
+ */
+ @Before
+ public void before() throws Exception {
+ m_clustersMock = createNiceMock(Clusters.class);
+ m_ambariMetaInfoMock = createNiceMock(AmbariMetaInfo.class);
+
+ MockModule mockModule = new MockModule();
+
+ // create an injector which will inject the mocks
+ m_injector = Guice.createInjector(mockModule);
+ }
+
+ /**
+ *
+ */
+ @After
+ public void teardown() {
+ }
+
+ /**
+ * Tests that properties which were explicitely removed in the current
+ * configurations by stack advisor are not re-introduced as "new" properties
+ * accidentally.
+ * <p/>
+ *
+ * HDP 2.4 defaults
+ * <ul>
+ * <li>foo-site/foo-property-1</li>
+ * <li>foo-site/foo-property-2</li>
+ * <li>bar-site/bar-property-1</li>
+ * </ul>
+ *
+ * HDP 2.5 defaults
+ * <ul>
+ * <li>foo-site/foo-property-1</li>
+ * <li>foo-site/foo-property-2</li>
+ * <li>bar-site/bar-property-1</li>
+ * <li>bar-site/bar-property-2</li>
+ * </ul>
+ *
+ * CURRENT 2.4 configs
+ * <ul>
+ * <li>foo-site/foo-property-1</li>
+ * <li>foo-site/foo-property-99</li>
+ * <li>bar-site/bar-property-1</li>
+ * <li>bar-site/bar-property-20</li>
+ * <li>bar-site/bar-property-99</li>
+ * </ul>
+ *
+ * The final merged configurations should detect that {{foo-property-2}}
+ * exists in both stacks but is not in the current configs and was therefore
+ * purposefully removed. It shoudl also detect that {{bar-property-20}} was
+ * added in the new stack and should be added in.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testMergedConfigurationsDoNotAddExplicitelyRemovedProperties() throws Exception {
+ Cluster cluster = createNiceMock(Cluster.class);
+ RepositoryVersionEntity repositoryVersionEntity = createNiceMock(RepositoryVersionEntity.class);
+ UpgradePack upgradePack = createNiceMock(UpgradePack.class);
+ StackEntity targetStack = createNiceMock(StackEntity.class);
+
+ // mocks which were bound previously
+ AmbariManagementController amc = m_injector.getInstance(AmbariManagementController.class);
+ AmbariMetaInfo ambariMetaInfo = m_injector.getInstance(AmbariMetaInfo.class);
+ ConfigHelper configHelper = m_injector.getInstance(ConfigHelper.class);
+ RepositoryVersionDAO repositoryVersionDAO = m_injector.getInstance(RepositoryVersionDAO.class);
+
+ EasyMock.expect(amc.getConfigHelper()).andReturn(configHelper);
+
+ EasyMock.expect(cluster.getCurrentStackVersion()).andReturn(s_currentStackId);
+ EasyMock.expect(cluster.getDesiredStackVersion()).andReturn(s_targetStackId);
+
+ EasyMock.expect(targetStack.getStackName()).andReturn("HDP").anyTimes();
+ EasyMock.expect(targetStack.getStackVersion()).andReturn("2.5").anyTimes();
+
+ EasyMock.expect(repositoryVersionEntity.getStack()).andReturn(targetStack);
+ EasyMock.expect(repositoryVersionDAO.findByStackNameAndVersion("HDP", "2.5.0.0-1234")).andReturn(repositoryVersionEntity);
+
+ EasyMock.expect(upgradePack.getGroups(Direction.UPGRADE)).andReturn(new ArrayList<Grouping>());
+
+ EasyMock.expect(ambariMetaInfo.getServices("HDP", "2.5")).andReturn(
+ new HashMap<String, ServiceInfo>());
+
+ // config helper mocks (the heart of this test)
+ Map<String, Map<String, String>> oldStackDefaultConfigurationsByType = new HashMap<>();
+ oldStackDefaultConfigurationsByType.put("foo-type", new HashMap<String, String>());
+ oldStackDefaultConfigurationsByType.get("foo-type").put("foo-property-1", "foo-value-1");
+ oldStackDefaultConfigurationsByType.get("foo-type").put("foo-property-2", "foo-value-2");
+ oldStackDefaultConfigurationsByType.put("bar-type", new HashMap<String, String>());
+ oldStackDefaultConfigurationsByType.get("bar-type").put("bar-property-1", "bar-value-1");
+
+ Map<String, Map<String, String>> newConfigurationsByType = new HashMap<>();
+ newConfigurationsByType.put("foo-type", new HashMap<String, String>());
+ newConfigurationsByType.get("foo-type").put("foo-property-1", "foo-value-1");
+ newConfigurationsByType.get("foo-type").put("foo-property-2", "foo-value-2");
+ newConfigurationsByType.put("bar-type", new HashMap<String, String>());
+ newConfigurationsByType.get("bar-type").put("bar-property-1", "bar-value-1");
+ newConfigurationsByType.get("bar-type").put("bar-property-20", "bar-value-20");
+
+ // HDP 2.4 configs
+ EasyMock.expect(configHelper.getDefaultProperties(EasyMock.eq(s_currentStackId),
+ EasyMock.anyObject(Cluster.class))).andReturn(oldStackDefaultConfigurationsByType);
+
+ // HDP 2.5 configs
+ EasyMock.expect(configHelper.getDefaultProperties(EasyMock.eq(s_targetStackId),
+ EasyMock.anyObject(Cluster.class))).andReturn(newConfigurationsByType);
+
+ // CURRENT HDP 2.4 configs
+ Config currentClusterConfigFoo = createNiceMock(Config.class);
+ Config currentClusterConfigBar = createNiceMock(Config.class);
+
+ Map<String, String> existingPropertiesFoo = new HashMap<>();
+ existingPropertiesFoo.put("foo-property-1", "foo-value-1");
+ existingPropertiesFoo.put("foo-property-99", "foo-value-99");
+ EasyMock.expect(currentClusterConfigFoo.getProperties()).andReturn(existingPropertiesFoo);
+
+ Map<String, String> existingPropertiesBar = new HashMap<>();
+ existingPropertiesBar.put("bar-property-1", "bar-value-1");
+ existingPropertiesBar.put("bar-property-99", "bar-value-99");
+ EasyMock.expect(currentClusterConfigBar.getProperties()).andReturn(existingPropertiesBar);
+
+ EasyMock.expect(cluster.getDesiredConfigByType("foo-type")).andReturn(currentClusterConfigFoo);
+ EasyMock.expect(cluster.getDesiredConfigByType("bar-type")).andReturn(currentClusterConfigBar);
+
+ // desired configs
+ Map<String, DesiredConfig> existingDesiredConfigurationsByType = new HashMap<>();
+ existingDesiredConfigurationsByType.put("foo-type", null);
+ existingDesiredConfigurationsByType.put("bar-type", null);
+ EasyMock.expect(cluster.getDesiredConfigs()).andReturn(existingDesiredConfigurationsByType);
+
+ // we need to know what configs are being created, so capture them
+ Capture<Map<String, Map<String, String>>> capturedArgument = EasyMock.newCapture();
+ configHelper.createConfigTypes(EasyMock.anyObject(Cluster.class),
+ EasyMock.anyObject(AmbariManagementController.class),
+ EasyMock.capture(capturedArgument),
+ EasyMock.anyString(), EasyMock.anyString());
+
+ EasyMock.expectLastCall();
+
+ replayAll();
+
+ UpgradeResourceProvider upgradeResourceProvider = new UpgradeResourceProvider(amc);
+ m_injector.injectMembers(upgradeResourceProvider);
+
+ upgradeResourceProvider.applyStackAndProcessConfigurations("HDP", cluster, "2.5.0.0-1234",
+ Direction.UPGRADE, upgradePack, "admin");
+
+ // assertion time!
+ Map<String, Map<String, String>> mergedConfigurations = capturedArgument.getValue();
+ Map<String, String> mergedFooSite = mergedConfigurations.get("foo-type");
+ Map<String, String> mergedBarSite = mergedConfigurations.get("bar-type");
+
+ // foo-site validation
+ Assert.assertEquals("foo-value-1", mergedFooSite.get("foo-property-1"));
+ Assert.assertEquals("foo-value-99", mergedFooSite.get("foo-property-99"));
+ Assert.assertFalse(mergedFooSite.containsKey("foo-property-2"));
+
+ // bar-site validation
+ Assert.assertEquals("bar-value-1", mergedBarSite.get("bar-property-1"));
+ Assert.assertEquals("bar-value-20", mergedBarSite.get("bar-property-20"));
+ Assert.assertEquals("bar-value-99", mergedBarSite.get("bar-property-99"));
+ Assert.assertEquals(3, mergedBarSite.size());
+ }
+
+
+ private class MockModule implements Module {
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void configure(Binder binder) {
+ binder.bind(AmbariMetaInfo.class).toInstance(m_ambariMetaInfoMock);
+ binder.bind(Clusters.class).toInstance(createNiceMock(Clusters.class));
+ binder.bind(OsFamily.class).toInstance(createNiceMock(OsFamily.class));
+ binder.bind(DBAccessor.class).toInstance(createNiceMock(DBAccessor.class));
+ binder.bind(EntityManager.class).toInstance(createNiceMock(EntityManager.class));
+ binder.bind(ActionManager.class).toInstance(createNiceMock(ActionManager.class));
+ binder.bind(HostRoleCommandDAO.class).toInstance(createNiceMock(HostRoleCommandDAO.class));
+ binder.bind(AmbariManagementController.class).toInstance(createNiceMock(AmbariManagementController.class));
+ binder.bind(ClusterController.class).toInstance(createNiceMock(ClusterController.class));
+ binder.bind(StackManagerFactory.class).toInstance(createNiceMock(StackManagerFactory.class));
+ binder.bind(SessionManager.class).toInstance(createNiceMock(SessionManager.class));
+ binder.bind(RequestExecutionFactory.class).toInstance(createNiceMock(RequestExecutionFactory.class));
+ binder.bind(ExecutionScheduler.class).toInstance(createNiceMock(ExecutionScheduler.class));
+ binder.bind(RequestFactory.class).toInstance(createNiceMock(RequestFactory.class));
+ binder.bind(StageFactory.class).toInstance(createNiceMock(StageFactory.class));
+ binder.bind(RoleGraphFactory.class).toInstance(createNiceMock(RoleGraphFactoryImpl.class));
+ binder.bind(AbstractRootServiceResponseFactory.class).toInstance(createNiceMock(AbstractRootServiceResponseFactory.class));
+ binder.bind(ConfigFactory.class).toInstance(createNiceMock(ConfigFactory.class));
+ binder.bind(ConfigGroupFactory.class).toInstance(createNiceMock(ConfigGroupFactory.class));
+ binder.bind(ServiceFactory.class).toInstance(createNiceMock(ServiceFactory.class));
+ binder.bind(ServiceComponentFactory.class).toInstance(createNiceMock(ServiceComponentFactory.class));
+ binder.bind(ServiceComponentHostFactory.class).toInstance(createNiceMock(ServiceComponentHostFactory.class));
+ binder.bind(PasswordEncoder.class).toInstance(createNiceMock(PasswordEncoder.class));
+ binder.bind(KerberosHelper.class).toInstance(createNiceMock(KerberosHelper.class));
+ binder.bind(Users.class).toInstance(createNiceMock(Users.class));
+ binder.bind(ConfigHelper.class).toInstance(createNiceMock(ConfigHelper.class));
+ binder.bind(RepositoryVersionDAO.class).toInstance(createNiceMock(RepositoryVersionDAO.class));
+
+ binder.requestStaticInjection(UpgradeResourceProvider.class);
+ }
+ }
+}