You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2016/09/16 18:18:18 UTC
ambari git commit: AMBARI-18391. Prerequisite checks should only
check services that are getting upgraded (ncole)
Repository: ambari
Updated Branches:
refs/heads/branch-dev-patch-upgrade 9dbcac6d5 -> e2085dd6f
AMBARI-18391. Prerequisite checks should only check services that are getting upgraded (ncole)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e2085dd6
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e2085dd6
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e2085dd6
Branch: refs/heads/branch-dev-patch-upgrade
Commit: e2085dd6fdeb7add7eacbccc293fe4df7a617fe7
Parents: 9dbcac6
Author: Nate Cole <nc...@hortonworks.com>
Authored: Fri Sep 16 10:18:28 2016 -0400
Committer: Nate Cole <nc...@hortonworks.com>
Committed: Fri Sep 16 14:17:44 2016 -0400
----------------------------------------------------------------------
.../server/checks/AbstractCheckDescriptor.java | 30 +++++-
.../orm/entities/RepositoryVersionEntity.java | 10 +-
.../state/repository/VersionDefinitionXml.java | 22 ++++
.../checks/AbstractCheckDescriptorTest.java | 104 +++++++++++++------
...duce2JobHistoryStatePreservingCheckTest.java | 23 +++-
.../ServicesNamenodeTruncateCheckTest.java | 20 +++-
6 files changed, 164 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/e2085dd6/ambari-server/src/main/java/org/apache/ambari/server/checks/AbstractCheckDescriptor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/AbstractCheckDescriptor.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/AbstractCheckDescriptor.java
index 4bc64d4..5b9cd00 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/checks/AbstractCheckDescriptor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/AbstractCheckDescriptor.java
@@ -22,23 +22,27 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.ambari.annotations.Experimental;
+import org.apache.ambari.annotations.ExperimentalFeature;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
-import org.apache.ambari.server.controller.PrereqCheckRequest;
import org.apache.ambari.server.configuration.Configuration;
+import org.apache.ambari.server.controller.PrereqCheckRequest;
import org.apache.ambari.server.orm.dao.ClusterVersionDAO;
import org.apache.ambari.server.orm.dao.HostVersionDAO;
import org.apache.ambari.server.orm.dao.RepositoryVersionDAO;
import org.apache.ambari.server.orm.dao.UpgradeDAO;
+import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.Config;
import org.apache.ambari.server.state.DesiredConfig;
+import org.apache.ambari.server.state.RepositoryType;
import org.apache.ambari.server.state.ServiceInfo;
-import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.stack.PrereqCheckType;
import org.apache.ambari.server.state.stack.PrerequisiteCheck;
import org.apache.ambari.server.state.stack.upgrade.RepositoryVersionHelper;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -120,6 +124,7 @@ public abstract class AbstractCheckDescriptor {
* @throws org.apache.ambari.server.AmbariException
* if server error happens
*/
+ @Experimental(feature = ExperimentalFeature.PATCH_UPGRADES)
public boolean isApplicable(PrereqCheckRequest request, List<String> requiredServices, boolean requiredAll) throws AmbariException {
final Cluster cluster = clustersProvider.get().getCluster(request.getClusterName());
Set<String> services = cluster.getServices().keySet();
@@ -137,6 +142,25 @@ public abstract class AbstractCheckDescriptor {
}
}
+ // !!! service is found and deployed - now check if it is part of the VDF
+ if (serviceFound && null != request.getTargetStackId()) {
+ String stackName = request.getTargetStackId().getStackName();
+ RepositoryVersionEntity rve = repositoryVersionDaoProvider.get().
+ findByStackNameAndVersion(stackName, request.getRepositoryVersion());
+
+ if (RepositoryType.STANDARD != rve.getType()) {
+ try {
+ Set<String> availableServices = rve.getRepositoryXml().getAvailableServiceNames();
+
+ if (!CollectionUtils.containsAny(availableServices, requiredServices)) {
+ serviceFound = false;
+ }
+ } catch (Exception e) {
+ LOG.warn("Could not parse xml for %s", request.getRepositoryVersion(), e);
+ }
+ }
+ }
+
return serviceFound;
}
@@ -290,7 +314,7 @@ public abstract class AbstractCheckDescriptor {
public Boolean isRequired(){
return getClass().getAnnotation(UpgradeCheck.class).required();
}
-
+
/**
* Return a boolean indicating whether or not configs allow bypassing errors during the RU/EU PreChecks.
* @return
http://git-wip-us.apache.org/repos/asf/ambari/blob/e2085dd6/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java
index 25aa62b..6950e40 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java
@@ -17,6 +17,8 @@
*/
package org.apache.ambari.server.orm.entities;
+import static java.util.Arrays.asList;
+
import java.util.Collections;
import java.util.List;
import java.util.Set;
@@ -38,12 +40,12 @@ import javax.persistence.NamedQueries;
import javax.persistence.NamedQuery;
import javax.persistence.OneToMany;
import javax.persistence.OneToOne;
+import javax.persistence.PrePersist;
+import javax.persistence.PreUpdate;
import javax.persistence.Table;
import javax.persistence.TableGenerator;
import javax.persistence.Transient;
import javax.persistence.UniqueConstraint;
-import javax.persistence.PreUpdate;
-import javax.persistence.PrePersist;
import org.apache.ambari.server.StaticallyInject;
import org.apache.ambari.server.state.RepositoryType;
@@ -58,8 +60,6 @@ import org.slf4j.LoggerFactory;
import com.google.inject.Inject;
import com.google.inject.Provider;
-import static java.util.Arrays.asList;
-
@Entity
@Table(name = "repo_version", uniqueConstraints = {
@UniqueConstraint(columnNames = {"display_name"}),
@@ -354,7 +354,7 @@ public class RepositoryVersionEntity {
* @return the XSD name extracted from the XML.
*/
public String getVersionXsd() {
- return versionXml;
+ return versionXsd;
}
/**
http://git-wip-us.apache.org/repos/asf/ambari/blob/e2085dd6/ambari-server/src/main/java/org/apache/ambari/server/state/repository/VersionDefinitionXml.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/repository/VersionDefinitionXml.java b/ambari-server/src/main/java/org/apache/ambari/server/state/repository/VersionDefinitionXml.java
index 6d5f23e..d246601 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/repository/VersionDefinitionXml.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/repository/VersionDefinitionXml.java
@@ -121,6 +121,7 @@ public class VersionDefinitionXml {
* are specified as "available".
*/
public Collection<AvailableService> getAvailableServices(StackInfo stack) {
+
if (null == m_availableMap) {
Map<String, ManifestService> manifests = buildManifest();
m_availableMap = new HashMap<>();
@@ -143,6 +144,27 @@ public class VersionDefinitionXml {
}
/**
+ * Gets the set of services that are included in this XML
+ * @return an empty set for STANDARD repositories, or a non-empty set for PATCH type.
+ */
+ public Set<String> getAvailableServiceNames() {
+ if (availableServices.isEmpty()) {
+ return Collections.emptySet();
+ } else {
+ Set<String> serviceNames = new HashSet<>();
+
+ Map<String, ManifestService> manifest = buildManifest();
+
+ for (AvailableServiceReference ref : availableServices) {
+ ManifestService ms = manifest.get(ref.serviceIdReference);
+ serviceNames.add(ms.serviceName);
+ }
+
+ return serviceNames;
+ }
+ }
+
+ /**
* Gets if the version definition was built as the default for a stack
* @return {@code true} if default for a stack
*/
http://git-wip-us.apache.org/repos/asf/ambari/blob/e2085dd6/ambari-server/src/test/java/org/apache/ambari/server/checks/AbstractCheckDescriptorTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/checks/AbstractCheckDescriptorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/checks/AbstractCheckDescriptorTest.java
index 55c8815..21878a0 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/checks/AbstractCheckDescriptorTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/checks/AbstractCheckDescriptorTest.java
@@ -17,42 +17,36 @@
*/
package org.apache.ambari.server.checks;
+import static org.easymock.EasyMock.anyString;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.reset;
+
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
-import junit.framework.Assert;
-import static org.easymock.EasyMock.anyString;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.expectLastCall;
-import static org.easymock.EasyMock.replay;
-
import org.apache.ambari.server.AmbariException;
-import org.apache.ambari.server.api.services.AmbariMetaInfo;
-import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.controller.PrereqCheckRequest;
-import org.apache.ambari.server.orm.dao.ClusterVersionDAO;
-import org.apache.ambari.server.orm.dao.HostVersionDAO;
import org.apache.ambari.server.orm.dao.RepositoryVersionDAO;
-import org.apache.ambari.server.orm.dao.UpgradeDAO;
+import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.RepositoryType;
import org.apache.ambari.server.state.Service;
-import org.apache.ambari.server.state.ServiceImpl;
+import org.apache.ambari.server.state.StackId;
+import org.apache.ambari.server.state.repository.VersionDefinitionXml;
import org.apache.ambari.server.state.stack.PrereqCheckType;
import org.apache.ambari.server.state.stack.PrerequisiteCheck;
-import org.apache.ambari.server.state.stack.upgrade.RepositoryVersionHelper;
import org.apache.ambari.server.state.stack.upgrade.UpgradeType;
-import org.apache.commons.collections.map.HashedMap;
import org.easymock.EasyMock;
+import org.junit.Assert;
import org.junit.Test;
-import com.google.inject.AbstractModule;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
import com.google.inject.Provider;
/**
@@ -60,6 +54,7 @@ import com.google.inject.Provider;
*/
public class AbstractCheckDescriptorTest {
final private Clusters clusters = EasyMock.createNiceMock(Clusters.class);
+ private final RepositoryVersionDAO repositoryVersionDao = EasyMock.createNiceMock(RepositoryVersionDAO.class);
private class TestCheckImpl extends AbstractCheckDescriptor {
private PrereqCheckType m_type;
@@ -74,6 +69,13 @@ public class AbstractCheckDescriptorTest {
return clusters;
}
};
+
+ repositoryVersionDaoProvider = new Provider<RepositoryVersionDAO>() {
+ @Override
+ public RepositoryVersionDAO get() {
+ return repositoryVersionDao;
+ }
+ };
}
@Override
@@ -135,20 +137,14 @@ public class AbstractCheckDescriptorTest {
AbstractCheckDescriptor check = new TestCheckImpl(PrereqCheckType.SERVICE);
PrereqCheckRequest request = new PrereqCheckRequest(clusterName, UpgradeType.ROLLING);
- List<String> oneServiceList = new ArrayList<String>() {{
- add("SERVICE1");
- }};
- List<String> atLeastOneServiceList = new ArrayList<String>() {{
- add("SERVICE1");
- add("NON_EXISTED_SERVICE");
- }};
- List<String> allServicesList = new ArrayList<String>(){{
- add("SERVICE1");
- add("SERVICE2");
- }};
- List<String> nonExistedList = new ArrayList<String>(){{
- add("NON_EXISTED_SERVICE");
- }};
+ List<String> oneServiceList = Arrays.asList("SERVICE1");
+
+ List<String> atLeastOneServiceList = Arrays.asList("SERVICE1", "NON_EXISTED_SERVICE");
+
+ List<String> allServicesList = Arrays.asList("SERVICE1", "SERVICE2");
+
+ List<String> nonExistedList = Arrays.asList("NON_EXISTED_SERVICE");
+
// case, where we need at least one service to be present
Assert.assertEquals(true, check.isApplicable(request, oneServiceList, false));
@@ -167,4 +163,50 @@ public class AbstractCheckDescriptorTest {
Assert.assertEquals(false, check.isApplicable(request, nonExistedList, true));
}
+ @Test
+ public void testIsApplicableWithVDF() throws Exception{
+ final String clusterName = "c1";
+ final Cluster cluster = EasyMock.createMock(Cluster.class);
+
+ Map<String, Service> services = new HashMap<String, Service>(){{
+ put("SERVICE1", null);
+ put("SERVICE2", null);
+ put("SERVICE3", null);
+ }};
+
+ expect(clusters.getCluster(anyString())).andReturn(cluster).atLeastOnce();
+ expect(cluster.getServices()).andReturn(services).atLeastOnce();
+
+ RepositoryVersionEntity repoVersion = EasyMock.createMock(RepositoryVersionEntity.class);
+ VersionDefinitionXml repoXml = EasyMock.createMock(VersionDefinitionXml.class);
+ expect(repoVersion.getType()).andReturn(RepositoryType.PATCH).atLeastOnce();
+ expect(repoVersion.getRepositoryXml()).andReturn(repoXml).atLeastOnce();
+ expect(repoXml.getAvailableServiceNames()).andReturn(Collections.singleton("SERVICE2")).atLeastOnce();
+
+ expect(repositoryVersionDao.findByStackNameAndVersion(
+ anyString(), anyString())).andReturn(repoVersion).atLeastOnce();
+
+ replay(clusters, cluster, repositoryVersionDao, repoVersion, repoXml);
+
+ AbstractCheckDescriptor check = new TestCheckImpl(PrereqCheckType.SERVICE);
+ PrereqCheckRequest request = new PrereqCheckRequest(clusterName, UpgradeType.ROLLING);
+ request.setTargetStackId(new StackId("HDP-2.5"));
+
+ List<String> allServicesList = Arrays.asList("SERVICE1", "SERVICE2");
+
+ // SERVICE2 is the only thing in VDF
+ Assert.assertEquals(true, check.isApplicable(request, allServicesList, false));
+
+ List<String> oneServicesList = Arrays.asList("SERVICE1");
+
+ // SERVICE2 is the only thing in VDF, a check for only SERVICE1 fail
+ Assert.assertEquals(false, check.isApplicable(request, oneServicesList, false));
+
+ // a VDF without available services is technically invalid, so expect any passed services to return false
+ reset(repoXml);
+ expect(repoXml.getAvailableServiceNames()).andReturn(Collections.<String>emptySet()).atLeastOnce();
+ replay(repoXml);
+ Assert.assertEquals(false, check.isApplicable(request, allServicesList, false));
+ }
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/e2085dd6/ambari-server/src/test/java/org/apache/ambari/server/checks/MapReduce2JobHistoryStatePreservingCheckTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/checks/MapReduce2JobHistoryStatePreservingCheckTest.java b/ambari-server/src/test/java/org/apache/ambari/server/checks/MapReduce2JobHistoryStatePreservingCheckTest.java
index bfe0c3e..12fdd69 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/checks/MapReduce2JobHistoryStatePreservingCheckTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/checks/MapReduce2JobHistoryStatePreservingCheckTest.java
@@ -17,33 +17,36 @@
*/
package org.apache.ambari.server.checks;
-import com.google.inject.Provider;
+import java.util.HashMap;
+import java.util.Map;
+
import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.controller.PrereqCheckRequest;
+import org.apache.ambari.server.orm.dao.RepositoryVersionDAO;
import org.apache.ambari.server.orm.entities.ClusterVersionEntity;
import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.Config;
import org.apache.ambari.server.state.DesiredConfig;
+import org.apache.ambari.server.state.RepositoryType;
import org.apache.ambari.server.state.Service;
import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.stack.PrereqCheckStatus;
import org.apache.ambari.server.state.stack.PrerequisiteCheck;
-import org.apache.ambari.server.state.stack.UpgradePack.PrerequisiteCheckConfig;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
-import java.util.HashMap;
-import java.util.Map;
+import com.google.inject.Provider;
/**
* Tests for {@link org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheckTest}
*/
public class MapReduce2JobHistoryStatePreservingCheckTest {
private final Clusters m_clusters = Mockito.mock(Clusters.class);
+ private final RepositoryVersionDAO m_repositoryVersionDao = Mockito.mock(RepositoryVersionDAO.class);
private final MapReduce2JobHistoryStatePreservingCheck m_check = new MapReduce2JobHistoryStatePreservingCheck();
@@ -59,8 +62,20 @@ public class MapReduce2JobHistoryStatePreservingCheckTest {
return m_clusters;
}
};
+
+ m_check.repositoryVersionDaoProvider = new Provider<RepositoryVersionDAO>() {
+ @Override
+ public RepositoryVersionDAO get() {
+ return m_repositoryVersionDao;
+ };
+ };
+
Configuration config = Mockito.mock(Configuration.class);
m_check.config = config;
+
+ RepositoryVersionEntity rve = Mockito.mock(RepositoryVersionEntity.class);
+ Mockito.when(rve.getType()).thenReturn(RepositoryType.STANDARD);
+ Mockito.when(m_repositoryVersionDao.findByStackNameAndVersion(Mockito.anyString(), Mockito.anyString())).thenReturn(rve);
}
/**
http://git-wip-us.apache.org/repos/asf/ambari/blob/e2085dd6/ambari-server/src/test/java/org/apache/ambari/server/checks/ServicesNamenodeTruncateCheckTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/checks/ServicesNamenodeTruncateCheckTest.java b/ambari-server/src/test/java/org/apache/ambari/server/checks/ServicesNamenodeTruncateCheckTest.java
index 87d4167..a2bd1a6 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/checks/ServicesNamenodeTruncateCheckTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/checks/ServicesNamenodeTruncateCheckTest.java
@@ -27,21 +27,23 @@ import java.util.Map;
import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.controller.PrereqCheckRequest;
+import org.apache.ambari.server.orm.dao.RepositoryVersionDAO;
+import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.Config;
+import org.apache.ambari.server.state.RepositoryType;
import org.apache.ambari.server.state.Service;
import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.stack.PrereqCheckStatus;
import org.apache.ambari.server.state.stack.PrerequisiteCheck;
-import org.apache.ambari.server.state.stack.UpgradePack.PrerequisiteCheckConfig;
import org.easymock.EasyMock;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
import com.google.inject.Provider;
-import org.mockito.Mockito;
/**
* Unit tests for ServicesNamenodeTruncateCheck
@@ -52,6 +54,7 @@ public class ServicesNamenodeTruncateCheckTest {
private Clusters m_clusters = EasyMock.createMock(Clusters.class);
private ServicesNamenodeTruncateCheck m_check = new ServicesNamenodeTruncateCheck();
private final Map<String, String> m_configMap = new HashMap<String, String>();
+ private RepositoryVersionDAO m_repositoryVersionDAO = EasyMock.createMock(RepositoryVersionDAO.class);
@Before
public void setup() throws Exception {
@@ -81,6 +84,19 @@ public class ServicesNamenodeTruncateCheckTest {
return m_clusters;
}
};
+
+ m_check.repositoryVersionDaoProvider = new Provider<RepositoryVersionDAO>() {
+ @Override
+ public RepositoryVersionDAO get() {
+ return m_repositoryVersionDAO;
+ }
+ };
+
+ RepositoryVersionEntity rve = EasyMock.createMock(RepositoryVersionEntity.class);
+ expect(rve.getType()).andReturn(RepositoryType.STANDARD).anyTimes();
+ expect(m_repositoryVersionDAO.findByStackNameAndVersion(EasyMock.anyString(), EasyMock.anyString())).andReturn(rve).anyTimes();
+ replay(m_repositoryVersionDAO, rve);
+
}