You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2016/02/24 16:08:05 UTC
[18/50] [abbrv] ambari git commit: AMBARI-14924. Create patch upgrade
orchestration based on available services (ncole)
AMBARI-14924. Create patch upgrade orchestration based on available services (ncole)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a2561302
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a2561302
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a2561302
Branch: refs/heads/trunk
Commit: a2561302530c859f27c678920014db4e4c4a5f7d
Parents: 0f9da42
Author: Nate Cole <nc...@hortonworks.com>
Authored: Thu Feb 4 13:57:18 2016 -0500
Committer: Nate Cole <nc...@hortonworks.com>
Committed: Fri Feb 5 08:50:13 2016 -0500
----------------------------------------------------------------------
.../checks/HostsRepositoryVersionCheck.java | 3 +-
.../internal/UpgradeResourceProvider.java | 35 +++++++++++-
.../VersionDefinitionResourceProvider.java | 6 +--
.../orm/entities/RepositoryVersionEntity.java | 2 +-
.../ambari/server/state/UpgradeContext.java | 28 ++++++++++
.../ambari/server/state/UpgradeHelper.java | 4 ++
.../checks/HostsRepositoryVersionCheckTest.java | 57 ++++++++++++++++++++
.../ambari/server/state/UpgradeHelperTest.java | 57 +++++++++++++++++++-
8 files changed, 184 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/a2561302/ambari-server/src/main/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheck.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheck.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheck.java
index 30250ef..620c651 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheck.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheck.java
@@ -53,6 +53,7 @@ public class HostsRepositoryVersionCheck extends AbstractCheckDescriptor {
super(CheckDescription.HOSTS_REPOSITORY_VERSION);
}
+ @Override
public boolean isApplicable(PrereqCheckRequest request) throws AmbariException {
return super.isApplicable(request) && request.getRepositoryVersion() != null;
}
@@ -77,7 +78,7 @@ public class HostsRepositoryVersionCheck extends AbstractCheckDescriptor {
for (HostVersionEntity hve : hostVersionDaoProvider.get().findByHost(host.getHostName())) {
if (hve.getRepositoryVersion().getVersion().equals(request.getRepositoryVersion())
- && hve.getState() == RepositoryVersionState.INSTALLED) {
+ && (hve.getState() == RepositoryVersionState.INSTALLED || hve.getState() == RepositoryVersionState.NOT_REQUIRED)) {
found = true;
break;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/a2561302/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
index 70440fc..db8c079 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
@@ -24,6 +24,7 @@ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.VERSION;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
@@ -36,8 +37,6 @@ import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import com.google.inject.assistedinject.Assisted;
-import com.google.inject.persist.Transactional;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.Role;
import org.apache.ambari.server.RoleCommand;
@@ -86,12 +85,15 @@ import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.Config;
import org.apache.ambari.server.state.ConfigHelper;
import org.apache.ambari.server.state.DesiredConfig;
+import org.apache.ambari.server.state.RepositoryType;
import org.apache.ambari.server.state.ServiceInfo;
import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.StackInfo;
import org.apache.ambari.server.state.UpgradeContext;
import org.apache.ambari.server.state.UpgradeHelper;
import org.apache.ambari.server.state.UpgradeHelper.UpgradeGroupHolder;
+import org.apache.ambari.server.state.repository.AvailableService;
+import org.apache.ambari.server.state.repository.VersionDefinitionXml;
import org.apache.ambari.server.state.stack.ConfigUpgradePack;
import org.apache.ambari.server.state.stack.PrereqCheckStatus;
import org.apache.ambari.server.state.stack.UpgradePack;
@@ -115,6 +117,8 @@ import com.google.common.collect.Lists;
import com.google.gson.Gson;
import com.google.inject.Inject;
import com.google.inject.Provider;
+import com.google.inject.assistedinject.Assisted;
+import com.google.inject.persist.Transactional;
/**
* Manages the ability to start and get status of upgrades.
@@ -701,12 +705,37 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
StackId sourceStackId = null;
StackId targetStackId = null;
+ Set<String> supportedServices = new HashSet<>();
+
switch (direction) {
case UPGRADE:
sourceStackId = cluster.getCurrentStackVersion();
RepositoryVersionEntity targetRepositoryVersion = s_repoVersionDAO.findByStackNameAndVersion(
sourceStackId.getStackName(), version);
+
+ EnumSet<RepositoryType> serviceAware = EnumSet.of(RepositoryType.PATCH, RepositoryType.SERVICE);
+ if (serviceAware.contains(targetRepositoryVersion.getType())) {
+
+ VersionDefinitionXml xml = null;
+ StackInfo stackInfo = s_metaProvider.get().getStack(sourceStackId.getStackName(),
+ sourceStackId.getStackVersion());
+
+ try {
+ xml = targetRepositoryVersion.getRepositoryXml();
+ } catch (Exception e) {
+ throw new AmbariException(String.format("Could not load repository definition for version %s", version));
+ }
+
+ if (null != xml) {
+ Collection<AvailableService> services = xml.getAvailableServices(stackInfo);
+
+ for (AvailableService available : services) {
+ supportedServices.add(available.getName());
+ }
+ }
+ }
+
targetStackId = targetRepositoryVersion.getStackId();
break;
case DOWNGRADE:
@@ -717,6 +746,8 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
UpgradeContext ctx = new UpgradeContext(resolver, sourceStackId, targetStackId, version,
direction, pack.getType());
+ ctx.setSupportedServices(supportedServices);
+
if (direction.isDowngrade()) {
if (requestMap.containsKey(UPGRADE_FROM_VERSION)) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/a2561302/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java
index ee99a1e..3ab5169 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java
@@ -262,7 +262,7 @@ public class VersionDefinitionResourceProvider extends AbstractAuthorizedResourc
List<RepositoryVersionEntity> entities = s_repoVersionDAO.findByStack(entity.getStackId());
if (entities.isEmpty()) {
- throw new AmbariException(String.format("Patch %s was uploaded, but there are no repositories for %s",
+ throw new IllegalArgumentException(String.format("Patch %s was uploaded, but there are no repositories for %s",
entity.getVersion(), entity.getStackId().toString()));
}
@@ -290,7 +290,7 @@ public class VersionDefinitionResourceProvider extends AbstractAuthorizedResourc
if (matching.isEmpty()) {
String format = "No versions matched pattern %s";
- throw new AmbariException(String.format(format,
+ throw new IllegalArgumentException(String.format(format,
emptyCompatible ? holder.xml.release.version : holder.xml.release.compatibleWith));
} else if (matching.size() > 1) {
Set<String> versions= new HashSet<>();
@@ -298,7 +298,7 @@ public class VersionDefinitionResourceProvider extends AbstractAuthorizedResourc
versions.add(match.getVersion());
}
- throw new AmbariException(String.format("More than one repository matches patch %s: %s",
+ throw new IllegalArgumentException(String.format("More than one repository matches patch %s: %s",
entity.getVersion(), StringUtils.join(versions, ", ")));
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/a2561302/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java
index 4af4216..3398709 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java
@@ -349,7 +349,7 @@ public class RepositoryVersionEntity {
/**
* Parse the version XML into its object representation. This causes the XML to be lazy-loaded
* from storage.
- * @return {@code null} if the XSD is not available.
+ * @return {@code null} if the XSD (from the XML) is not available.
* @throws Exception
*/
public VersionDefinitionXml getRepositoryXml() throws Exception {
http://git-wip-us.apache.org/repos/asf/ambari/blob/a2561302/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java
index bd87a55..05aecac 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java
@@ -19,8 +19,10 @@ package org.apache.ambari.server.state;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.stack.MasterHostResolver;
@@ -84,6 +86,8 @@ public class UpgradeContext {
*/
private boolean m_autoSkipManualVerification = false;
+ private Set<String> m_supported = new HashSet<>();
+
/**
* Constructor.
*
@@ -350,4 +354,28 @@ public class UpgradeContext {
public void setAutoSkipManualVerification(boolean autoSkipManualVerification) {
m_autoSkipManualVerification = autoSkipManualVerification;
}
+
+ /**
+ * Sets the service names that are supported by an upgrade. This is used for
+ * {@link RepositoryType#PATCH} and {@link RepositoryType#SERVICE}.
+ *
+ * @param services the set of specific services
+ */
+ public void setSupportedServices(Set<String> services) {
+ m_supported = services;
+ }
+
+ /**
+ * Gets if a service is supported. If there are no services marked for the context,
+ * then ALL services are supported
+ * @param serviceName the service name to check.
+ * @return {@code true} when the service is supported
+ */
+ public boolean isServiceSupported(String serviceName) {
+ if (m_supported.isEmpty() || m_supported.contains(serviceName)) {
+ return true;
+ }
+
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/a2561302/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
index 8213a78..0d9176d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
@@ -317,6 +317,10 @@ public class UpgradeHelper {
// !!! cluster and service checks are empty here
for (UpgradePack.OrderService service : services) {
+ if (!context.isServiceSupported(service.serviceName)) {
+ continue;
+ }
+
if (upgradePack.getType() == UpgradeType.ROLLING && !allTasks.containsKey(service.serviceName)) {
continue;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/a2561302/ambari-server/src/test/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheckTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheckTest.java b/ambari-server/src/test/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheckTest.java
index 4529554..433eee1 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheckTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheckTest.java
@@ -207,4 +207,61 @@ public class HostsRepositoryVersionCheckTest {
Assert.assertEquals(PrereqCheckStatus.PASS, check.getStatus());
}
+ @Test
+ public void testPerformWithVersionNotRequired() throws Exception {
+ final HostsRepositoryVersionCheck hostsRepositoryVersionCheck = new HostsRepositoryVersionCheck();
+ hostsRepositoryVersionCheck.clustersProvider = new Provider<Clusters>() {
+
+ @Override
+ public Clusters get() {
+ return clusters;
+ }
+ };
+ hostsRepositoryVersionCheck.repositoryVersionDaoProvider = new Provider<RepositoryVersionDAO>() {
+ @Override
+ public RepositoryVersionDAO get() {
+ return repositoryVersionDAO;
+ }
+ };
+ hostsRepositoryVersionCheck.hostVersionDaoProvider = new Provider<HostVersionDAO>() {
+ @Override
+ public HostVersionDAO get() {
+ return hostVersionDAO;
+ }
+ };
+
+ final Cluster cluster = Mockito.mock(Cluster.class);
+ Mockito.when(cluster.getClusterId()).thenReturn(1L);
+ Mockito.when(cluster.getDesiredStackVersion()).thenReturn(new StackId());
+ Mockito.when(clusters.getCluster("cluster")).thenReturn(cluster);
+ final Map<String, Host> hosts = new HashMap<String, Host>();
+ final Host host1 = Mockito.mock(Host.class);
+ final Host host2 = Mockito.mock(Host.class);
+ final Host host3 = Mockito.mock(Host.class);
+ Mockito.when(host1.getMaintenanceState(1L)).thenReturn(MaintenanceState.OFF);
+ Mockito.when(host2.getMaintenanceState(1L)).thenReturn(MaintenanceState.OFF);
+ Mockito.when(host3.getMaintenanceState(1L)).thenReturn(MaintenanceState.OFF);
+ hosts.put("host1", host1);
+ hosts.put("host2", host2);
+ hosts.put("host3", host3);
+ Mockito.when(clusters.getHostsForCluster("cluster")).thenReturn(hosts);
+
+ RepositoryVersionEntity rve = new RepositoryVersionEntity();
+ rve.setVersion("1.1.1");
+
+ HostVersionEntity hve = new HostVersionEntity();
+ hve.setRepositoryVersion(rve);
+ hve.setState(RepositoryVersionState.NOT_REQUIRED);
+
+ Mockito.when(
+ hostVersionDAO.findByHost(Mockito.anyString())).thenReturn(
+ Collections.singletonList(hve));
+
+ PrerequisiteCheck check = new PrerequisiteCheck(null, null);
+ PrereqCheckRequest request = new PrereqCheckRequest("cluster");
+ request.setRepositoryVersion("1.1.1");
+ hostsRepositoryVersionCheck.perform(check, request);
+ Assert.assertEquals(PrereqCheckStatus.PASS, check.getStatus());
+ }
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/a2561302/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
index eb5bf62..b15157e 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
@@ -61,10 +61,12 @@ import org.apache.ambari.server.state.stack.upgrade.StageWrapper;
import org.apache.ambari.server.state.stack.upgrade.Task;
import org.apache.ambari.server.state.stack.upgrade.TaskWrapper;
import org.apache.ambari.server.state.stack.upgrade.UpgradeType;
+import org.apache.ambari.server.utils.EventBusSynchronizer;
import org.easymock.EasyMock;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.springframework.security.core.context.SecurityContextHolder;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
@@ -74,7 +76,6 @@ import com.google.inject.Injector;
import com.google.inject.Module;
import com.google.inject.persist.PersistService;
import com.google.inject.util.Modules;
-import org.springframework.security.core.context.SecurityContextHolder;
/**
* Tests the {@link UpgradeHelper} class
@@ -123,6 +124,8 @@ public class UpgradeHelperTest {
// create an injector which will inject the mocks
injector = Guice.createInjector(Modules.override(injectorModule).with(mockModule));
injector.getInstance(GuiceJpaInitializer.class);
+ EventBusSynchronizer.synchronizeAmbariEventPublisher(injector);
+ EventBusSynchronizer.synchronizeAlertEventPublisher(injector);
helper = injector.getInstance(OrmTestHelper.class);
ambariMetaInfo = injector.getInstance(AmbariMetaInfo.class);
@@ -257,6 +260,58 @@ public class UpgradeHelperTest {
}
@Test
+ public void testSupportedServiceUpgradeOrchestration() throws Exception {
+ Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("foo", "bar");
+ assertTrue(upgrades.isEmpty());
+
+ upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1");
+
+ ServiceInfo si = ambariMetaInfo.getService("HDP", "2.1.1", "ZOOKEEPER");
+ si.setDisplayName("Zk");
+ ComponentInfo ci = si.getComponentByName("ZOOKEEPER_SERVER");
+ ci.setDisplayName("ZooKeeper1 Server2");
+
+ assertTrue(upgrades.containsKey("upgrade_test"));
+ UpgradePack upgrade = upgrades.get("upgrade_test");
+ assertNotNull(upgrade);
+
+ makeCluster();
+
+ UpgradeContext context = new UpgradeContext(m_masterHostResolver, HDP_21,
+ HDP_21, UPGRADE_VERSION, Direction.UPGRADE, UpgradeType.ROLLING);
+ context.setSupportedServices(Collections.singleton("ZOOKEEPER"));
+
+
+ List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context);
+
+ assertEquals(3, groups.size());
+
+ assertEquals("PRE_CLUSTER", groups.get(0).name);
+ assertEquals("ZOOKEEPER", groups.get(1).name);
+ assertEquals("POST_CLUSTER", groups.get(2).name);
+
+
+ UpgradeGroupHolder group = groups.get(1);
+ // check that the display name is being used
+ assertTrue(group.items.get(1).getText().contains("ZooKeeper1 Server2"));
+ assertEquals(group.items.get(5).getText(), "Service Check Zk");
+
+ UpgradeGroupHolder postGroup = groups.get(2);
+ assertEquals("POST_CLUSTER", postGroup.name);
+ assertEquals("Finalize Upgrade", postGroup.title);
+ assertEquals(3, postGroup.items.size());
+ assertEquals("Confirm Finalize", postGroup.items.get(0).getText());
+ assertEquals("Execute HDFS Finalize", postGroup.items.get(1).getText());
+ assertEquals("Save Cluster State", postGroup.items.get(2).getText());
+ assertEquals(StageWrapper.Type.SERVER_SIDE_ACTION, postGroup.items.get(2).getType());
+
+ assertEquals(4, groups.get(0).items.size());
+ assertEquals(6, groups.get(1).items.size());
+ assertEquals(3, groups.get(2).items.size());
+ }
+
+
+ @Test
public void testUpgradeServerActionOrchestration() throws Exception {
Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1");