You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2014/05/02 23:16:18 UTC
git commit: AMBARI-5661. YARN and MR2 failed to start after upgrade.
(swagle)
Repository: ambari
Updated Branches:
refs/heads/branch-1.6.0 0fac3de4d -> 775397514
AMBARI-5661. YARN and MR2 failed to start after upgrade. (swagle)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/77539751
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/77539751
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/77539751
Branch: refs/heads/branch-1.6.0
Commit: 775397514f98e6f8e76ed14311aa3e3ff0baa25f
Parents: 0fac3de
Author: Siddharth Wagle <sw...@hortonworks.com>
Authored: Fri May 2 14:15:38 2014 -0700
Committer: Siddharth Wagle <sw...@hortonworks.com>
Committed: Fri May 2 14:15:38 2014 -0700
----------------------------------------------------------------------
.../server/upgrade/AbstractUpgradeCatalog.java | 79 +++++++++++++++++-
.../server/upgrade/UpgradeCatalog160.java | 6 +-
.../server/upgrade/UpgradeCatalog160Test.java | 30 +++++--
.../server/upgrade/UpgradeCatalogTest.java | 85 +++++++++++++++++++-
4 files changed, 188 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/77539751/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
index 86229a7..393eca5 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
@@ -23,18 +23,21 @@ import com.google.inject.Provider;
import com.google.inject.persist.Transactional;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.configuration.Configuration;
+import org.apache.ambari.server.controller.AmbariManagementController;
+import org.apache.ambari.server.controller.ConfigurationRequest;
import org.apache.ambari.server.orm.DBAccessor;
import org.apache.ambari.server.orm.dao.MetainfoDAO;
import org.apache.ambari.server.orm.entities.MetainfoEntity;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Config;
import org.apache.ambari.server.utils.VersionUtils;
-import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.persistence.EntityManager;
-import java.io.File;
-import java.io.IOException;
import java.sql.SQLException;
+import java.text.MessageFormat;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Map;
@@ -168,6 +171,76 @@ public abstract class AbstractUpgradeCatalog implements UpgradeCatalog {
dbAccessor.executeQuery(String.format("ALTER ROLE %s SET search_path to '%s';", dbUser, dbName));
}
+ /**
+ * Create a new cluster scoped configuration with the new properties added
+ * to the existing set of properties.
+ * @param configType Configuration type. (hdfs-site, etc.)
+ * @param properties Map of key value pairs to add / update.
+ */
+ protected void updateConfigurationProperties(String configType,
+ Map<String, String> properties, boolean updateIfExists) throws
+ AmbariException {
+ AmbariManagementController controller = injector.getInstance(AmbariManagementController.class);
+ String newTag = "version" + System.currentTimeMillis();
+
+ Clusters clusters = controller.getClusters();
+ if (clusters == null) {
+ return;
+ }
+ Map<String, Cluster> clusterMap = clusters.getClusters();
+
+ if (clusterMap != null && !clusterMap.isEmpty()) {
+ for (Cluster cluster : clusterMap.values()) {
+ Config oldConfig = cluster.getDesiredConfigByType(configType);
+
+ if (properties != null) {
+ Map<String, Config> all = cluster.getConfigsByType(configType);
+ if (all == null || !all.containsKey(newTag) || properties.size() > 0) {
+
+ Map<String, String> mergedProperties =
+ mergeProperties(oldConfig.getProperties(), properties, updateIfExists);
+
+ LOG.info("Applying configuration with tag ''{0}'' to " +
+ "cluster ''{1}''", newTag, cluster.getClusterName());
+
+ ConfigurationRequest cr = new ConfigurationRequest();
+ cr.setClusterName(cluster.getClusterName());
+ cr.setVersionTag(newTag);
+ cr.setType(configType);
+ cr.setProperties(mergedProperties);
+ controller.createConfiguration(cr);
+
+ Config baseConfig = cluster.getConfig(cr.getType(), cr.getVersionTag());
+ if (baseConfig != null) {
+ String authName = "ambari-upgrade";
+
+ if (cluster.addDesiredConfig(authName, baseConfig)) {
+ LOG.info("cluster '" + cluster.getClusterName() + "' "
+ + "changed by: '" + authName + "'; "
+ + "type='" + baseConfig.getType() + "' "
+ + "tag='" + baseConfig.getVersionTag() + "'"
+ + " from='"+ oldConfig.getVersionTag() + "'");
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ private Map<String, String> mergeProperties(Map<String, String> originalProperties,
+ Map<String, String> newProperties,
+ boolean updateIfExists) {
+
+ Map<String, String> properties = new HashMap<String, String>(originalProperties);
+ for (Map.Entry<String, String> entry : newProperties.entrySet()) {
+ if (!properties.containsKey(entry.getKey()) || updateIfExists) {
+ properties.put(entry.getKey(), entry.getValue());
+ }
+ }
+ return properties;
+ }
+
@Override
public void upgradeSchema() throws AmbariException, SQLException {
if (getDbType().equals(Configuration.POSTGRES_DB_NAME)) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/77539751/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog160.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog160.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog160.java
index 5318e22..c86c3ed 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog160.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog160.java
@@ -26,6 +26,7 @@ import org.apache.ambari.server.orm.DBAccessor;
import java.sql.SQLException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
/**
@@ -97,7 +98,7 @@ public class UpgradeCatalog160 extends AbstractUpgradeCatalog {
protected void executeDMLUpdates() throws AmbariException, SQLException {
String dbType = getDbType();
- //add new sequences for view entity
+ // Add new sequences for view entity
String valueColumnName = "\"value\"";
if (Configuration.ORACLE_DB_NAME.equals(dbType) || Configuration.MYSQL_DB_NAME.equals(dbType)) {
valueColumnName = "value";
@@ -105,6 +106,9 @@ public class UpgradeCatalog160 extends AbstractUpgradeCatalog {
dbAccessor.executeQuery("INSERT INTO ambari_sequences(sequence_name, " + valueColumnName + ") " +
"VALUES('viewentity_id_seq', 0)", true);
+
+ // Add missing property for YARN
+ updateConfigurationProperties("global", Collections.singletonMap("jobhistory_heapsize", "900"), false);
}
@Override
http://git-wip-us.apache.org/repos/asf/ambari/blob/77539751/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog160Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog160Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog160Test.java
index 062338b..c02b109 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog160Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog160Test.java
@@ -23,23 +23,32 @@ import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Module;
import org.apache.ambari.server.configuration.Configuration;
+import org.apache.ambari.server.controller.AmbariManagementController;
import org.apache.ambari.server.orm.DBAccessor;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
import org.easymock.Capture;
import org.junit.Assert;
import org.junit.Test;
import java.lang.reflect.Field;
+import java.lang.reflect.Method;
import java.sql.SQLException;
+import java.util.Collections;
import java.util.List;
+import java.util.Map;
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertFalse;
import static junit.framework.Assert.assertTrue;
import static junit.framework.Assert.assertNull;
import static org.easymock.EasyMock.capture;
+import static org.easymock.EasyMock.createMockBuilder;
+import static org.easymock.EasyMock.createNiceControl;
import static org.easymock.EasyMock.createNiceMock;
import static org.easymock.EasyMock.eq;
import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
import static org.easymock.EasyMock.replay;
import static org.easymock.EasyMock.verify;
@@ -79,23 +88,34 @@ public class UpgradeCatalog160Test {
@Test
public void testExecuteDMLUpdates() throws Exception {
-
Configuration configuration = createNiceMock(Configuration.class);
+ DBAccessor dbAccessor = createNiceMock(DBAccessor.class);
+
+ Method m = AbstractUpgradeCatalog.class.getDeclaredMethod
+ ("updateConfigurationProperties", String.class, Map.class, boolean.class);
+
+ UpgradeCatalog160 upgradeCatalog = createMockBuilder(UpgradeCatalog160.class)
+ .addMockedMethod(m).createMock();
+
expect(configuration.getDatabaseUrl()).andReturn(Configuration.JDBC_IN_MEMORY_URL).anyTimes();
- final DBAccessor dbAccessor = createNiceMock(DBAccessor.class);
- UpgradeCatalog160 upgradeCatalog = (UpgradeCatalog160) getUpgradeCatalog(dbAccessor);
+ upgradeCatalog.updateConfigurationProperties("global",
+ Collections.singletonMap("jobhistory_heapsize", "900"), false);
+ expectLastCall();
- replay(dbAccessor, configuration);
+ replay(upgradeCatalog, dbAccessor, configuration);
Class<?> c = AbstractUpgradeCatalog.class;
Field f = c.getDeclaredField("configuration");
f.setAccessible(true);
f.set(upgradeCatalog, configuration);
+ f = c.getDeclaredField("dbAccessor");
+ f.setAccessible(true);
+ f.set(upgradeCatalog, dbAccessor);
upgradeCatalog.executeDMLUpdates();
- verify(dbAccessor, configuration);
+ verify(upgradeCatalog, dbAccessor, configuration);
}
@Test
http://git-wip-us.apache.org/repos/asf/ambari/blob/77539751/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogTest.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogTest.java
index df4b712..322b3f1 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogTest.java
@@ -25,20 +25,34 @@ import com.google.inject.persist.PersistService;
import junit.framework.Assert;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
+import org.apache.ambari.server.controller.AmbariManagementController;
+import org.apache.ambari.server.controller.ClusterRequest;
+import org.apache.ambari.server.controller.ConfigurationRequest;
import org.apache.ambari.server.orm.GuiceJpaInitializer;
import org.apache.ambari.server.orm.InMemoryDefaultTestModule;
+import org.apache.ambari.server.orm.dao.ClusterDAO;
+import org.apache.ambari.server.orm.entities.ClusterEntity;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Config;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import java.sql.SQLException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
public class UpgradeCatalogTest {
private Injector injector;
private AmbariMetaInfo metaInfo;
+ private final String DESIRED_STACK_VERSION = "{\"stackName\":\"HDP\"," +
+ "\"stackVersion\":\"1.2.0\"}";
private static class UpgradeCatalog149 extends AbstractUpgradeCatalog {
@@ -88,11 +102,9 @@ public class UpgradeCatalogTest {
injector.getInstance(PersistService.class).stop();
}
-
@Test
public void testUpgradePath() throws Exception {
- SchemaUpgradeHelper schemaUpgradeHelper = injector.getInstance
- (SchemaUpgradeHelper.class);
+ SchemaUpgradeHelper schemaUpgradeHelper = injector.getInstance(SchemaUpgradeHelper.class);
Set<UpgradeCatalog> upgradeCatalogSet = schemaUpgradeHelper.getAllUpgradeCatalogs();
@@ -107,4 +119,71 @@ public class UpgradeCatalogTest {
Assert.assertEquals("1.4.9", upgradeCatalogs.get(0).getTargetVersion());
Assert.assertEquals("1.5.0", upgradeCatalogs.get(1).getTargetVersion());
}
+
+ @Test
+ public void testUpdateConfigurationProperties() throws Exception {
+ AmbariManagementController controller = injector.getInstance(AmbariManagementController.class);
+ ClusterRequest r = new ClusterRequest(null, "c1", "HDP-0.1", null);
+ controller.createCluster(r);
+ Clusters clusters = injector.getInstance(Clusters.class);
+ Cluster cluster = clusters.getCluster("c1");
+
+ Assert.assertNotNull(cluster);
+
+ Map<String, String> properties = new HashMap<String, String>() {{
+ put("a", "b");
+ }};
+
+ final ClusterRequest cl = new ClusterRequest(
+ cluster.getClusterId(),
+ cluster.getClusterName(),
+ cluster.getDesiredStackVersion().getStackVersion(),
+ null
+ );
+
+ ConfigurationRequest cr = new ConfigurationRequest();
+ cr.setClusterName(cluster.getClusterName());
+ cr.setType("global");
+ cr.setVersionTag("version1");
+ cr.setProperties(properties);
+
+ cl.setDesiredConfig(cr);
+
+ controller.updateClusters(new HashSet<ClusterRequest>() {{ add(cl); }}, null);
+
+ Config config = cluster.getConfig("global", "version1");
+
+ Assert.assertNotNull(config);
+ Assert.assertEquals(properties, config.getProperties());
+
+ // Add new
+ UpgradeCatalog149 testCatalog = injector.getInstance(UpgradeCatalog149.class);
+ testCatalog.updateConfigurationProperties("global",
+ Collections.singletonMap("x", "y"), false);
+ config = cluster.getDesiredConfigByType("global");
+ String version = config.getVersionTag();
+ Assert.assertNotNull(config);
+ Assert.assertNotSame("version1", version);
+ Assert.assertTrue(config.getProperties().containsKey("x"));
+ Assert.assertEquals("y", config.getProperties().get("x"));
+
+ // Override value
+ testCatalog.updateConfigurationProperties("global",
+ Collections.singletonMap("x", "z"), true);
+ config = cluster.getDesiredConfigByType("global");
+ Assert.assertNotNull(config);
+ Assert.assertNotSame(version, config.getVersionTag());
+ Assert.assertTrue(config.getProperties().containsKey("x"));
+ Assert.assertEquals("z", config.getProperties().get("x"));
+ version = config.getVersionTag();
+
+ // Retain original
+ testCatalog.updateConfigurationProperties("global",
+ Collections.singletonMap("x", "y"), false);
+ config = cluster.getDesiredConfigByType("global");
+ Assert.assertNotNull(config);
+ Assert.assertNotSame(version, config.getVersionTag());
+ Assert.assertTrue(config.getProperties().containsKey("x"));
+ Assert.assertEquals("z", config.getProperties().get("x"));
+ }
}