You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jl...@apache.org on 2017/07/23 23:34:10 UTC
[37/50] [abbrv] ambari git commit: AMBARI-21526 - ZKFC Doesn't Update
its Version On Some Upgrades (jonathanhurley)
AMBARI-21526 - ZKFC Doesn't Update its Version On Some Upgrades (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4fdca575
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4fdca575
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4fdca575
Branch: refs/heads/branch-feature-AMBARI-14714
Commit: 4fdca575bcf61e5d8d85a4ff6b7fc08d61dfb5e4
Parents: 6b4d093
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Wed Jul 19 16:49:01 2017 -0400
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Wed Jul 19 20:30:27 2017 -0400
----------------------------------------------------------------------
.../libraries/functions/stack_features.py | 2 +-
.../listeners/upgrade/StackVersionListener.java | 33 +++++++++--
.../2.1.0.2.0/package/scripts/zkfc_slave.py | 8 +--
.../upgrade/StackVersionListenerTest.java | 58 ++++++++++++--------
4 files changed, 65 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/4fdca575/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py
index 24201dd..b241288 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py
@@ -123,7 +123,7 @@ def get_stack_feature_version(config):
# if this is not an upgrade, then we take the simple path
if upgrade_direction is None:
Logger.info(
- "Stack Feature Version Info: Cluster Stack={0}, Cluster Current Version={1}, Command Stack={2}, Command Version={3}-> {4}".format(
+ "Stack Feature Version Info: Cluster Stack={0}, Cluster Current Version={1}, Command Stack={2}, Command Version={3} -> {4}".format(
stack_version, current_cluster_version, command_stack, command_version, version_for_stack_feature_checks))
return version_for_stack_feature_checks
http://git-wip-us.apache.org/repos/asf/ambari/blob/4fdca575/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java b/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java
index f8d5a5f..b812476 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java
@@ -19,13 +19,16 @@ package org.apache.ambari.server.events.listeners.upgrade;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.EagerSingleton;
+import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.events.HostComponentVersionAdvertisedEvent;
import org.apache.ambari.server.events.publishers.VersionEventPublisher;
import org.apache.ambari.server.orm.dao.RepositoryVersionDAO;
import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.ComponentInfo;
import org.apache.ambari.server.state.ServiceComponent;
import org.apache.ambari.server.state.ServiceComponentHost;
+import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.State;
import org.apache.ambari.server.state.UpgradeState;
import org.apache.commons.lang.StringUtils;
@@ -34,6 +37,7 @@ import org.slf4j.LoggerFactory;
import com.google.common.eventbus.Subscribe;
import com.google.inject.Inject;
+import com.google.inject.Provider;
import com.google.inject.Singleton;
/**
@@ -55,6 +59,13 @@ public class StackVersionListener {
private RepositoryVersionDAO repositoryVersionDAO;
/**
+ * Used for looking up a component's advertising version status given a stack
+ * and name.
+ */
+ @Inject
+ private Provider<AmbariMetaInfo> ambariMetaInfoProvider;
+
+ /**
* Constructor.
*
* @param eventPublisher the publisher
@@ -95,20 +106,30 @@ public class StackVersionListener {
// Update host component version value if needed
try {
- ServiceComponent sc = cluster.getService(sch.getServiceName()).getServiceComponent(
- sch.getServiceComponentName());
+ // get the component information for the desired stack; if a component
+ // moves from UNKNOWN to providing a version, we must do the version
+ // advertised check against the target stack
+ StackId desiredStackId = sch.getDesiredStackId();
+
+ AmbariMetaInfo ambariMetaInfo = ambariMetaInfoProvider.get();
+ ComponentInfo componentInfo = ambariMetaInfo.getComponent(desiredStackId.getStackName(),
+ desiredStackId.getStackVersion(), sch.getServiceName(), sch.getServiceComponentName());
// not advertising a version, do nothing
- if (!sc.isVersionAdvertised()) {
+ if (!componentInfo.isVersionAdvertised()) {
// that's odd; a version came back - log it and still do nothing
if (!StringUtils.equalsIgnoreCase(UNKNOWN_VERSION, newVersion)) {
- LOG.debug(
+ LOG.warn(
"ServiceComponent {} doesn't advertise version, however ServiceHostComponent {} on host {} advertised version as {}. Skipping version update",
- sc.getName(), sch.getServiceComponentName(), sch.getHostName(), newVersion);
+ sch.getServiceComponentName(), sch.getServiceComponentName(), sch.getHostName(),
+ newVersion);
}
return;
}
+ ServiceComponent sc = cluster.getService(sch.getServiceName()).getServiceComponent(
+ sch.getServiceComponentName());
+
// proces the UNKNOWN version
if (StringUtils.equalsIgnoreCase(UNKNOWN_VERSION, newVersion)) {
processUnknownDesiredVersion(cluster, sc, sch, newVersion);
@@ -215,4 +236,4 @@ public class StackVersionListener {
sch.setUpgradeState(upgradeState);
sch.recalculateHostVersionState();
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/4fdca575/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py
index ca5f605..cd47109 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py
@@ -30,11 +30,6 @@ from resource_management.core import shell
from resource_management.libraries.functions import conf_select, stack_select
from resource_management.libraries.functions.constants import StackFeature
from resource_management.libraries.functions.check_process_status import check_process_status
-from resource_management.libraries.functions.security_commons import build_expectations
-from resource_management.libraries.functions.security_commons import cached_kinit_executor
-from resource_management.libraries.functions.security_commons import get_params_from_filesystem
-from resource_management.libraries.functions.security_commons import validate_security_config_properties
-from resource_management.libraries.functions.security_commons import FILE_TYPE_XML
from resource_management.libraries.functions.stack_features import check_stack_feature
from resource_management.libraries.script import Script
from resource_management.core.resources.zkmigrator import ZkMigrator
@@ -144,8 +139,7 @@ class ZkfcSlaveDefault(ZkfcSlave):
Logger.info("Executing Stack Upgrade pre-restart")
import params
env.set_params(params)
- if params.version and check_stack_feature(StackFeature.ZKFC_VERSION_ADVERTISED, params.version) \
- and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version):
+ if check_stack_feature(StackFeature.ZKFC_VERSION_ADVERTISED, params.version_for_stack_feature_checks):
conf_select.select(params.stack_name, "hadoop", params.version)
stack_select.select("hadoop-hdfs-zkfc", params.version)
http://git-wip-us.apache.org/repos/asf/ambari/blob/4fdca575/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListenerTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListenerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListenerTest.java
index efe594a..ff6fab9 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListenerTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListenerTest.java
@@ -65,16 +65,15 @@ public class StackVersionListenerTest extends EasyMockSupport {
private static final RepositoryVersionEntity DUMMY_REPOSITORY_VERSION_ENTITY = new RepositoryVersionEntity();
private static final HostVersionEntity DUMMY_HOST_VERSION_ENTITY = new HostVersionEntity();
private static final UpgradeEntity DUMMY_UPGRADE_ENTITY = new UpgradeEntity();
- public static final String STACK_NAME = "HDP-2.4.0.0";
- public static final String STACK_VERSION = "2.4.0.0";
+ public static final String STACK_NAME = "HDP";
+ public static final String STACK_VERSION = "2.4";
private Cluster cluster;
private ServiceComponentHost sch;
private Service service;
private ServiceComponent serviceComponent;
private VersionEventPublisher publisher = new VersionEventPublisher();
- private ComponentInfo componentInfo;
- private StackId stackId;
+ private StackId stackId = new StackId(STACK_NAME, STACK_VERSION);
@TestSubject
private StackVersionListener listener = new StackVersionListener(publisher);
@@ -82,6 +81,12 @@ public class StackVersionListenerTest extends EasyMockSupport {
@Mock
private Provider<AmbariMetaInfo> ambariMetaInfoProvider;
+ @Mock
+ private ComponentInfo componentInfo;
+
+ @Mock
+ private AmbariMetaInfo ambariMetaInfo;
+
@Before
public void setup() throws Exception {
cluster = createNiceMock(Cluster.class);
@@ -89,20 +94,27 @@ public class StackVersionListenerTest extends EasyMockSupport {
service = createNiceMock(Service.class);
serviceComponent = createNiceMock(ServiceComponent.class);
componentInfo = createNiceMock(ComponentInfo.class);
- stackId = createNiceMock(StackId.class);
expect(cluster.getClusterId()).andReturn(CLUSTER_ID);
- expect(cluster.getService(SERVICE_NAME)).andReturn(service).atLeastOnce();
- expect(service.getServiceComponent(SERVICE_COMPONENT_NAME)).andReturn(serviceComponent).atLeastOnce();
+ expect(cluster.getService(SERVICE_NAME)).andReturn(service).anyTimes();
+ expect(service.getServiceComponent(SERVICE_COMPONENT_NAME)).andReturn(
+ serviceComponent).anyTimes();
+ expect(sch.getDesiredStackId()).andReturn(stackId).atLeastOnce();
expect(sch.getServiceName()).andReturn(SERVICE_NAME).atLeastOnce();
expect(sch.getServiceComponentName()).andReturn(SERVICE_COMPONENT_NAME).atLeastOnce();
+
+ expect(ambariMetaInfoProvider.get()).andReturn(ambariMetaInfo).atLeastOnce();
+ expect(ambariMetaInfo.getComponent(STACK_NAME, STACK_VERSION, SERVICE_NAME,
+ SERVICE_COMPONENT_NAME)).andReturn(componentInfo).atLeastOnce();
+
+ injectMocks(listener);
}
@Test
public void testRecalculateHostVersionStateWhenVersionIsNullAndNewVersionIsNotBlank() throws AmbariException {
expect(sch.getVersion()).andReturn(null);
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE);
+ expect(componentInfo.isVersionAdvertised()).andReturn(true).once();
sch.setVersion(INVALID_NEW_VERSION);
expectLastCall().once();
expect(sch.recalculateHostVersionState()).andReturn(null).once();
@@ -115,7 +127,7 @@ public class StackVersionListenerTest extends EasyMockSupport {
@Test
public void testRecalculateHostVersionStateWhenVersionIsUnknownAndNewVersionIsNotBlank() throws AmbariException {
expect(sch.getVersion()).andReturn(UNKNOWN_VERSION);
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE);
+ expect(componentInfo.isVersionAdvertised()).andReturn(true).once();
sch.setVersion(INVALID_NEW_VERSION);
expectLastCall().once();
expect(sch.recalculateHostVersionState()).andReturn(null).once();
@@ -128,7 +140,7 @@ public class StackVersionListenerTest extends EasyMockSupport {
@Test
public void testRecalculateClusterVersionStateWhenVersionIsNullAndNewVersionIsValid() throws AmbariException {
expect(sch.getVersion()).andReturn(null);
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE);
+ expect(componentInfo.isVersionAdvertised()).andReturn(true).once();
sch.setVersion(VALID_NEW_VERSION);
expectLastCall().once();
expect(sch.recalculateHostVersionState()).andReturn(DUMMY_HOST_VERSION_ENTITY).once();
@@ -141,7 +153,7 @@ public class StackVersionListenerTest extends EasyMockSupport {
@Test
public void testRecalculateClusterVersionStateWhenVersionIsUnknownAndNewVersionIsValid() throws AmbariException {
expect(sch.getVersion()).andReturn(UNKNOWN_VERSION);
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE);
+ expect(componentInfo.isVersionAdvertised()).andReturn(true).once();
sch.setVersion(VALID_NEW_VERSION);
expectLastCall().once();
expect(sch.recalculateHostVersionState()).andReturn(DUMMY_HOST_VERSION_ENTITY).once();
@@ -154,7 +166,7 @@ public class StackVersionListenerTest extends EasyMockSupport {
@Test
public void testRecalculateHostVersionStateWhenComponentDesiredVersionIsUnknownAndNewVersionIsNotValid() throws AmbariException {
expect(serviceComponent.getDesiredVersion()).andReturn(UNKNOWN_VERSION);
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE);
+ expect(componentInfo.isVersionAdvertised()).andReturn(true).once();
sch.setUpgradeState(UpgradeState.NONE);
expectLastCall().once();
sch.setVersion(INVALID_NEW_VERSION);
@@ -169,7 +181,7 @@ public class StackVersionListenerTest extends EasyMockSupport {
@Test
public void testRecalculateClusterVersionStateWhenComponentDesiredVersionIsUnknownAndNewVersionIsValid() throws AmbariException {
expect(serviceComponent.getDesiredVersion()).andReturn(UNKNOWN_VERSION);
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE);
+ expect(componentInfo.isVersionAdvertised()).andReturn(true).once();
sch.setUpgradeState(UpgradeState.NONE);
expectLastCall().once();
sch.setVersion(VALID_NEW_VERSION);
@@ -183,7 +195,7 @@ public class StackVersionListenerTest extends EasyMockSupport {
@Test
public void testRecalculateClusterVersionStateWhenVersionNotAdvertised() throws AmbariException {
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.FALSE);
+ expect(componentInfo.isVersionAdvertised()).andReturn(false).once();
replayAll();
sendEventAndVerify(VALID_NEW_VERSION);
}
@@ -191,7 +203,7 @@ public class StackVersionListenerTest extends EasyMockSupport {
@Test
public void testNoActionTakenOnNullVersion() {
- expect(serviceComponent.isVersionAdvertised()).andReturn(true);
+ expect(componentInfo.isVersionAdvertised()).andReturn(true).once();
resetAll();
replayAll();
@@ -204,7 +216,7 @@ public class StackVersionListenerTest extends EasyMockSupport {
expect(sch.getVersion()).andReturn(VALID_PREVIOUS_VERSION);
expect(sch.getUpgradeState()).andReturn(UpgradeState.IN_PROGRESS);
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE);
+ expect(componentInfo.isVersionAdvertised()).andReturn(true).once();
sch.setUpgradeState(UpgradeState.COMPLETE);
expectLastCall().once();
@@ -218,7 +230,7 @@ public class StackVersionListenerTest extends EasyMockSupport {
public void testSetUpgradeStateToNoneWhenNoUpgradeAndNewVersionIsEqualToComponentDesiredVersion() {
expect(sch.getVersion()).andReturn(VALID_PREVIOUS_VERSION);
expect(sch.getUpgradeState()).andReturn(UpgradeState.IN_PROGRESS);
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE);
+ expect(componentInfo.isVersionAdvertised()).andReturn(true).once();
sch.setUpgradeState(UpgradeState.NONE);
expectLastCall().once();
@@ -232,7 +244,7 @@ public class StackVersionListenerTest extends EasyMockSupport {
public void testSetUpgradeStateToVersionMismatchWhenUpgradeIsInProgressAndNewVersionIsNotEqualToComponentDesiredVersion() {
expect(sch.getVersion()).andReturn(VALID_PREVIOUS_VERSION);
expect(sch.getUpgradeState()).andReturn(UpgradeState.IN_PROGRESS);
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE);
+ expect(componentInfo.isVersionAdvertised()).andReturn(true).once();
sch.setUpgradeState(UpgradeState.VERSION_MISMATCH);
expectLastCall().once();
@@ -251,7 +263,7 @@ public class StackVersionListenerTest extends EasyMockSupport {
expect(sch.getUpgradeState()).andReturn(UpgradeState.VERSION_MISMATCH);
expect(cluster.getUpgradeInProgress()).andReturn(DUMMY_UPGRADE_ENTITY);
expect(serviceComponent.getDesiredVersion()).andStubReturn(VALID_NEW_VERSION);
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE);
+ expect(componentInfo.isVersionAdvertised()).andReturn(true).once();
sch.setUpgradeState(UpgradeState.COMPLETE);
expectLastCall().once();
@@ -265,7 +277,7 @@ public class StackVersionListenerTest extends EasyMockSupport {
expect(sch.getVersion()).andReturn(VALID_PREVIOUS_VERSION);
expect(sch.getUpgradeState()).andReturn(UpgradeState.VERSION_MISMATCH);
expect(serviceComponent.getDesiredVersion()).andStubReturn(VALID_NEW_VERSION);
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE);
+ expect(componentInfo.isVersionAdvertised()).andReturn(true).once();
sch.setUpgradeState(UpgradeState.NONE);
expectLastCall().once();
@@ -277,7 +289,7 @@ public class StackVersionListenerTest extends EasyMockSupport {
@Test
public void testSetUpgradeStateToVersionMismatchByDefaultWhenHostAndNewVersionsAreValid() {
expect(sch.getVersion()).andReturn(VALID_PREVIOUS_VERSION);
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE);
+ expect(componentInfo.isVersionAdvertised()).andReturn(true).once();
sch.setUpgradeState(UpgradeState.VERSION_MISMATCH);
expectLastCall().once();
@@ -289,7 +301,7 @@ public class StackVersionListenerTest extends EasyMockSupport {
@Test
public void testSetRepositoryVersion() throws Exception {
expect(sch.getVersion()).andReturn(UNKNOWN_VERSION);
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE);
+ expect(componentInfo.isVersionAdvertised()).andReturn(true).once();
RepositoryVersionDAO dao = createNiceMock(RepositoryVersionDAO.class);
RepositoryVersionEntity entity = createNiceMock(RepositoryVersionEntity.class);
@@ -321,6 +333,8 @@ public class StackVersionListenerTest extends EasyMockSupport {
*/
@Test
public void testRepositoryVersionNotSetDuringUpgrade() throws Exception {
+ expect(componentInfo.isVersionAdvertised()).andReturn(true).once();
+
// this call will make it seem like there is an upgrade in progress
expect(cluster.getUpgradeInProgress()).andReturn(createNiceMock(UpgradeEntity.class));