You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ds...@apache.org on 2018/06/11 16:44:39 UTC
[ambari] branch trunk updated: [AMBARI-24050] Oozie service check failed during EU on unsecure clust… (#1483)
This is an automated email from the ASF dual-hosted git repository.
dsen pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 6fb7289 [AMBARI-24050] Oozie service check failed during EU on unsecure clust… (#1483)
6fb7289 is described below
commit 6fb7289a0f6e0e732f6bd647493f264ed8969d72
Author: Dmitry Sen <ds...@apache.org>
AuthorDate: Mon Jun 11 19:44:26 2018 +0300
[AMBARI-24050] Oozie service check failed during EU on unsecure clust… (#1483)
* [AMBARI-24050] Oozie service check failed during EU on unsecure cluster due to presence of "ELService" configs (dsen)
* [AMBARI-24050] Oozie service check failed during EU on unsecure cluster due to presence of "ELService" configs (dsen) - import order fix
* [AMBARI-24050] Oozie service check failed during EU on unsecure cluster due to presence of "ELService" configs (dsen) - changes according to review
---
.../upgrades/OozieConfigCalculation.java | 103 ++++++++++++--
.../server/state/ServiceComponentSupport.java | 5 +-
.../upgrades/OozieConfigCalculationTest.java | 153 +++++++++++++++++++++
3 files changed, 248 insertions(+), 13 deletions(-)
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/OozieConfigCalculation.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/OozieConfigCalculation.java
index 0ae5325..ac0454c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/OozieConfigCalculation.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/OozieConfigCalculation.java
@@ -18,6 +18,7 @@
package org.apache.ambari.server.serveraction.upgrades;
+import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentMap;
import java.util.regex.Matcher;
@@ -30,24 +31,103 @@ import org.apache.ambari.server.agent.CommandReport;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Config;
import org.apache.ambari.server.state.Host;
+import org.apache.ambari.server.state.ServiceComponentSupport;
+import org.apache.ambari.server.state.StackId;
+import org.apache.ambari.server.state.UpgradeContext;
+import org.apache.commons.collections.CollectionUtils;
+
+import com.google.inject.Inject;
/**
- * Changes oozie-env during upgrade (adds -Dhdp.version to $HADOOP_OPTS variable)
+ * Changes oozie-env (adds -Dhdp.version to $HADOOP_OPTS variable)
+ * and oozie-site (removes oozie.service.ELService.ext.functions.*) during upgrade
*/
public class OozieConfigCalculation extends AbstractUpgradeServerAction {
- private static final String TARGET_CONFIG_TYPE = "oozie-env";
+
+ private static final String FALCON_SERVICE_NAME = "FALCON";
+ @Inject
+ private ServiceComponentSupport serviceComponentSupport;
+
+ private static final String OOZIE_ENV_TARGET_CONFIG_TYPE = "oozie-env";
+ private static final String OOZIE_SITE_TARGET_CONFIG_TYPE = "oozie-site";
+ private static final String ELSERVICE_PROPERTIES_NAME_PREFIX = "oozie.service.ELService.ext.functions.";
private static final String CONTENT_PROPERTY_NAME = "content";
+ private boolean oozie_env_updated = false;
+ private boolean oozie_site_updated = false;
@Override
public CommandReport execute(ConcurrentMap<String, Object> requestSharedDataContext)
throws AmbariException, InterruptedException {
String clusterName = getExecutionCommand().getClusterName();
Cluster cluster = getClusters().getCluster(clusterName);
- Config config = cluster.getDesiredConfigByType(TARGET_CONFIG_TYPE);
+ StringBuilder stdOutBuilder = new StringBuilder();
- if (config == null) {
+ try {
+ changeOozieEnv(cluster, stdOutBuilder);
+ } catch (Exception e) {
return createCommandReport(0, HostRoleStatus.FAILED,"{}",
- String.format("Source type %s not found", TARGET_CONFIG_TYPE), "");
+ String.format("Source type %s not found", OOZIE_ENV_TARGET_CONFIG_TYPE), "");
+ }
+
+ UpgradeContext upgradeContext = getUpgradeContext(cluster);
+ StackId targetStackId = upgradeContext.getTargetStack();
+
+
+ if (!serviceComponentSupport.isServiceSupported(FALCON_SERVICE_NAME, targetStackId.getStackName(), targetStackId.getStackVersion())) {
+ try {
+ removeFalconPropertiesFromOozieSize(cluster, stdOutBuilder);
+ } catch (AmbariException e) {
+ return createCommandReport(0, HostRoleStatus.FAILED, "{}",
+ String.format("Source type %s not found", OOZIE_SITE_TARGET_CONFIG_TYPE), "");
+ }
+ }
+
+ if (oozie_env_updated || oozie_site_updated) {
+ agentConfigsHolder.updateData(cluster.getClusterId(), cluster.getHosts().stream().map(Host::getHostId).collect(Collectors.toList()));
+ }
+
+ return createCommandReport(0, HostRoleStatus.COMPLETED, "{}",
+ stdOutBuilder.toString(), "");
+ }
+
+ /**
+ * Changes oozie-site (removes oozie.service.ELService.ext.functions.*)
+ */
+ private void removeFalconPropertiesFromOozieSize(Cluster cluster, StringBuilder stringBuilder) throws AmbariException {
+ Config config = cluster.getDesiredConfigByType(OOZIE_SITE_TARGET_CONFIG_TYPE);
+
+ if (config == null) {
+ throw new AmbariException(String.format("Target config not found %s", OOZIE_SITE_TARGET_CONFIG_TYPE));
+ }
+
+ Map<String, String> properties = config.getProperties();
+ List<String> propertiesToRemove = properties.keySet().stream().filter(
+ s -> s.startsWith(ELSERVICE_PROPERTIES_NAME_PREFIX)).collect(Collectors.toList());
+
+ if (!CollectionUtils.isEmpty(propertiesToRemove)) {
+ stringBuilder.append(String.format("Removed following properties from %s: %s", OOZIE_SITE_TARGET_CONFIG_TYPE, propertiesToRemove));
+ stringBuilder.append(System.lineSeparator());
+ properties.keySet().removeAll(propertiesToRemove);
+ oozie_site_updated = true;
+ } else {
+ stringBuilder.append(String.format("No properties with prefix %s found in %s", ELSERVICE_PROPERTIES_NAME_PREFIX, OOZIE_ENV_TARGET_CONFIG_TYPE));
+ stringBuilder.append(System.lineSeparator());
+ return;
+ }
+
+ config.setProperties(properties);
+ config.save();
+ }
+
+ /**
+ * Changes oozie-env (adds -Dhdp.version to $HADOOP_OPTS variable)
+ */
+ private void changeOozieEnv(Cluster cluster, StringBuilder stringBuilder) throws AmbariException {
+
+ Config config = cluster.getDesiredConfigByType(OOZIE_ENV_TARGET_CONFIG_TYPE);
+
+ if (config == null) {
+ throw new AmbariException(String.format("Target config not found %s", OOZIE_ENV_TARGET_CONFIG_TYPE));
}
Map<String, String> properties = config.getProperties();
@@ -56,19 +136,18 @@ public class OozieConfigCalculation extends AbstractUpgradeServerAction {
String newContent = processPropertyValue(oldContent);
if (newContent.equals(oldContent)) {
- return createCommandReport(0, HostRoleStatus.COMPLETED, "{}",
- "-Dhdp.version option has been already added to $HADOOP_OPTS variable", "");
+ stringBuilder.append("-Dhdp.version option has been already added to $HADOOP_OPTS variable");
+ stringBuilder.append(System.lineSeparator());
+ return;
} else {
properties.put(CONTENT_PROPERTY_NAME, newContent);
+ oozie_env_updated = true;
+ stringBuilder.append(String.format("Added -Dhdp.version to $HADOOP_OPTS variable at %s", OOZIE_ENV_TARGET_CONFIG_TYPE));
+ stringBuilder.append(System.lineSeparator());
}
config.setProperties(properties);
config.save();
- agentConfigsHolder.updateData(cluster.getClusterId(), cluster.getHosts().stream().map(Host::getHostId).collect(Collectors.toList()));
-
- return createCommandReport(0, HostRoleStatus.COMPLETED, "{}",
- String.format("Added -Dhdp.version to $HADOOP_OPTS variable at %s", TARGET_CONFIG_TYPE), "");
-
}
public static String processPropertyValue(String oldContent) {
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentSupport.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentSupport.java
index d1aebac..9bb1ca4 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentSupport.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentSupport.java
@@ -56,7 +56,10 @@ public class ServiceComponentSupport {
.collect(toSet());
}
- private boolean isServiceSupported(String serviceName, String stackName, String stackVersion) {
+ /**
+ * Checks if the service is supported by the stack using metainfo
+ */
+ public boolean isServiceSupported(String serviceName, String stackName, String stackVersion) {
try {
ServiceInfo service = metaInfo.get().getServices(stackName, stackVersion).get(serviceName);
return service != null && !service.isDeleted();
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/OozieConfigCalculationTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/OozieConfigCalculationTest.java
index f2c7795..d2389f0 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/OozieConfigCalculationTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/OozieConfigCalculationTest.java
@@ -17,17 +17,170 @@
*/
package org.apache.ambari.server.serveraction.upgrades;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.eq;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import java.lang.reflect.Field;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper;
+import org.apache.ambari.server.actionmanager.HostRoleCommand;
+import org.apache.ambari.server.agent.CommandReport;
+import org.apache.ambari.server.agent.ExecutionCommand;
+import org.apache.ambari.server.agent.stomp.AgentConfigsHolder;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Config;
+import org.apache.ambari.server.state.ServiceComponentSupport;
+import org.apache.ambari.server.state.StackId;
+import org.apache.ambari.server.state.UpgradeContext;
import org.apache.commons.lang.StringUtils;
+import org.easymock.EasyMock;
+import org.junit.Before;
import org.junit.Test;
+import org.powermock.api.easymock.PowerMock;
+
+import com.google.inject.Injector;
+
+import junit.framework.Assert;
/**
* Tests OozieConfigCalculation logic
*/
public class OozieConfigCalculationTest {
+ private Injector m_injector;
+ private Clusters m_clusters;
+ private AgentConfigsHolder agentConfigsHolder;
+ private Field m_clusterField;
+ private Field m_componentSupportField;
+ private Field agentConfigsHolderField;
+ private OozieConfigCalculation m_action;
+ private final UpgradeContext m_mockUpgradeContext = EasyMock.createNiceMock(UpgradeContext.class);
+ private ServiceComponentSupport componentSupport;
+
+ @Before
+ public void setup() throws Exception {
+
+ m_injector = EasyMock.createMock(Injector.class);
+ componentSupport = EasyMock.createMock(ServiceComponentSupport.class);
+ m_clusters = EasyMock.createMock(Clusters.class);
+ agentConfigsHolder = createMock(AgentConfigsHolder.class);
+ Cluster cluster = EasyMock.createMock(Cluster.class);
+
+ m_action = PowerMock.createNicePartialMock(OozieConfigCalculation.class, "getUpgradeContext");
+
+ expect(m_action.getUpgradeContext(cluster)).andReturn(m_mockUpgradeContext).once();
+ StackId stackId = new StackId("HDP", "3.0.0");
+ expect(m_mockUpgradeContext.getTargetStack()).andReturn(stackId).once();
+
+ expect(componentSupport.isServiceSupported("FALCON", stackId.getStackName(), stackId.getStackVersion()))
+ .andReturn(false).once();
+
+ Map<String, String> siteMockProperties = new HashMap<String, String>() {{
+ put("oozie.service.ELService.ext.functions.coord-action-create", "some value");
+ put("oozie.service.ELService.ext.functions.coord-action-start", "some value");
+ put("oozie.systemmode", "NORMAL");
+ }};
+
+ Config oozie_site = EasyMock.createNiceMock(Config.class);
+ expect(oozie_site.getType()).andReturn("oozie-site").anyTimes();
+ expect(oozie_site.getProperties()).andReturn(siteMockProperties).anyTimes();
+ expect(cluster.getDesiredConfigByType("oozie-site")).andReturn(oozie_site).atLeastOnce();
+
+ Map<String, String> mockProperties = new HashMap<String, String>() {{
+ put("content", "#!/bin/bash\n" +
+ "\n" +
+ "if [ -d \"/usr/lib/bigtop-tomcat\" ]; then\n" +
+ " export OOZIE_CONFIG=${OOZIE_CONFIG:-/etc/oozie/conf}\n" +
+ " export CATALINA_BASE=${CATALINA_BASE:-{{oozie_server_dir}}}\n" +
+ " export CATALINA_TMPDIR=${CATALINA_TMPDIR:-/var/tmp/oozie}\n" +
+ " export OOZIE_CATALINA_HOME=/usr/lib/bigtop-tomcat\n" +
+ "fi\n" +
+ "\n" +
+ "# export OOZIE_BASE_URL=\"http://${OOZIE_HTTP_HOSTNAME}:${OOZIE_HTTP_PORT}/oozie\"\n" +
+ "export JAVA_LIBRARY_PATH={{hadoop_lib_home}}/native/Linux-amd64-64");
+ }};
+
+ Config oozieEnv = EasyMock.createNiceMock(Config.class);
+ expect(oozieEnv.getType()).andReturn("oozie-env").anyTimes();
+ expect(oozieEnv.getProperties()).andReturn(mockProperties).anyTimes();
+ expect(cluster.getDesiredConfigByType("oozie-env")).andReturn(oozieEnv).atLeastOnce();
+
+ expect(m_clusters.getCluster((String) anyObject())).andReturn(cluster).anyTimes();
+ expect(cluster.getClusterId()).andReturn(1L).atLeastOnce();
+ expect(cluster.getHosts()).andReturn(Collections.emptyList()).atLeastOnce();
+ agentConfigsHolder.updateData(eq(1L), eq(Collections.emptyList()));
+ expectLastCall().atLeastOnce();
+
+ expect(m_injector.getInstance(Clusters.class)).andReturn(m_clusters).atLeastOnce();
+
+ expect(m_injector.getInstance(ServiceComponentSupport.class)).andReturn(componentSupport).atLeastOnce();
+
+ replay(componentSupport, m_mockUpgradeContext, m_action, m_injector, m_clusters, cluster, oozieEnv, oozie_site, agentConfigsHolder);
+
+ m_clusterField = AbstractUpgradeServerAction.class.getDeclaredField("m_clusters");
+ m_clusterField.setAccessible(true);
+
+ m_componentSupportField = OozieConfigCalculation.class.getDeclaredField("serviceComponentSupport");
+ m_componentSupportField.setAccessible(true);
+
+ agentConfigsHolderField = AbstractUpgradeServerAction.class.getDeclaredField("agentConfigsHolder");
+ agentConfigsHolderField.setAccessible(true);
+ }
+
+ @Test
+ public void testAction() throws Exception {
+ Map<String, String> commandParams = new HashMap<>();
+ commandParams.put("clusterName", "c1");
+
+ ExecutionCommand executionCommand = new ExecutionCommand();
+ executionCommand.setCommandParams(commandParams);
+ executionCommand.setClusterName("c1");
+
+ HostRoleCommand hrc = EasyMock.createMock(HostRoleCommand.class);
+ expect(hrc.getRequestId()).andReturn(1L).anyTimes();
+ expect(hrc.getStageId()).andReturn(2L).anyTimes();
+ expect(hrc.getExecutionCommandWrapper()).andReturn(new ExecutionCommandWrapper(executionCommand)).anyTimes();
+ replay(hrc);
+
+ m_clusterField.set(m_action, m_clusters);
+ m_componentSupportField.set(m_action, componentSupport);
+ agentConfigsHolderField.set(m_action, agentConfigsHolder);
+
+ m_action.setExecutionCommand(executionCommand);
+ m_action.setHostRoleCommand(hrc);
+
+ CommandReport report = m_action.execute(null);
+ assertNotNull(report);
+
+ Cluster c = m_clusters.getCluster("c1");
+ Config oozieSiteConfig = c.getDesiredConfigByType("oozie-site");
+ Map<String, String> map = oozieSiteConfig.getProperties();
+
+ assertTrue(map.size()==1);
+ assertTrue(map.containsKey("oozie.systemmode"));
+ assertTrue(map.get("oozie.systemmode").equals("NORMAL"));
+
+ assertTrue(report.getStdOut().contains("Removed following properties"));
+
+ Config oozieEnvConfig = c.getDesiredConfigByType("oozie-env");
+ map = oozieEnvConfig.getProperties();
+
+ Assert.assertTrue(map.containsKey("content"));
+ String content = map.get("content");
+ assertTrue(content.endsWith("export HADOOP_OPTS=\"-Dhdp.version=$HDP_VERSION $HADOOP_OPTS\" "));
+ }
+
/**
* Checks that -Dhdp.version is added to $HADOOP_OPTS variable at oozie-env
* content.
--
To stop receiving notification emails like this one, please contact
dsen@apache.org.