You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ja...@apache.org on 2016/07/21 21:02:32 UTC
[17/19] ambari git commit: AMBARI-17798. RU/EU: Add falcon to oozie
admin user for HDP 2.5 (Venkat Ranganathan via alejandro)
AMBARI-17798. RU/EU: Add falcon to oozie admin user for HDP 2.5 (Venkat Ranganathan via alejandro)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/18ba883b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/18ba883b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/18ba883b
Branch: refs/heads/branch-embedded-views
Commit: 18ba883b314f2b8259a648b4ab1fd36f4a5cfda0
Parents: aff87ed
Author: Alejandro Fernandez <af...@hortonworks.com>
Authored: Thu Jul 21 13:44:47 2016 -0700
Committer: Alejandro Fernandez <af...@hortonworks.com>
Committed: Thu Jul 21 13:44:47 2016 -0700
----------------------------------------------------------------------
.../upgrades/FixOozieAdminUsers.java | 95 ++++++++++++
.../HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml | 1 +
.../stacks/HDP/2.3/upgrades/upgrade-2.5.xml | 2 +
.../HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml | 4 +-
.../stacks/HDP/2.4/upgrades/upgrade-2.5.xml | 2 +
.../stacks/HDP/2.5/services/stack_advisor.py | 47 ++++--
.../upgrades/FixOozieAdminUsersTest.java | 154 +++++++++++++++++++
7 files changed, 287 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/18ba883b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FixOozieAdminUsers.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FixOozieAdminUsers.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FixOozieAdminUsers.java
new file mode 100644
index 0000000..3a06476
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FixOozieAdminUsers.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.serveraction.upgrades;
+
+import com.google.inject.Inject;
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.actionmanager.HostRoleStatus;
+import org.apache.ambari.server.agent.CommandReport;
+import org.apache.ambari.server.serveraction.AbstractServerAction;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Config;
+import org.apache.commons.lang.StringUtils;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentMap;
+
+/**
+ * During stack upgrade, update lzo codec path in mapreduce.application.classpath and
+ * at tez.cluster.additional.classpath.prefix to look like
+ * /usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar
+ */
+public class FixOozieAdminUsers extends AbstractServerAction {
+ private static final String TARGET_OOZIE_CONFIG_TYPE = "oozie-env";
+ private static final String OOZIE_ADMIN_USERS_PROP = "oozie_admin_users";
+ private static final String FALCON_CONFIG_TYPE = "falcon-env";
+ private static final String FALCON_USER_PROP = "falcon_user";
+
+
+ @Inject
+ private Clusters clusters;
+
+ @Override
+ public CommandReport execute(ConcurrentMap<String, Object> requestSharedDataContext)
+ throws AmbariException, InterruptedException {
+ String clusterName = getExecutionCommand().getClusterName();
+ Cluster cluster = clusters.getCluster(clusterName);
+ Config oozieConfig = cluster.getDesiredConfigByType(TARGET_OOZIE_CONFIG_TYPE);
+ Config falconConfig = cluster.getDesiredConfigByType(FALCON_CONFIG_TYPE);
+
+ if (falconConfig == null) {
+ return createCommandReport(0, HostRoleStatus.COMPLETED, "{}",
+ "Falcon configuration unavailable", "");
+
+ }
+ if (oozieConfig == null) {
+ return createCommandReport(0, HostRoleStatus.COMPLETED, "{}",
+ "Oozie configuration unavailable", "");
+ }
+
+ Map<String, String> oozieProperties = oozieConfig.getProperties();
+ String currentOozieAdmins = oozieProperties.get(OOZIE_ADMIN_USERS_PROP);
+ if (currentOozieAdmins.isEmpty()) {
+ currentOozieAdmins = "";
+ }
+ Map<String, String> falconProperties = falconConfig.getProperties();
+ String falconUser = falconProperties.get(FALCON_USER_PROP);
+ if (StringUtils.isBlank(falconUser)) {
+ return createCommandReport(0, HostRoleStatus.COMPLETED, "{}",
+ "Falcon user not set", "");
+ }
+
+ if (currentOozieAdmins.indexOf(falconUser) >= 0) {
+ return createCommandReport(0, HostRoleStatus.COMPLETED, "{}",
+ "Falcon user already member of Oozie admins", "");
+ }
+
+ String newOozieAdminUsers = currentOozieAdmins + "," + falconUser;
+
+ oozieProperties.put(OOZIE_ADMIN_USERS_PROP, newOozieAdminUsers);
+
+ oozieConfig.setProperties(oozieProperties);
+ oozieConfig.persist(false);
+
+ return createCommandReport(0, HostRoleStatus.COMPLETED, "{}",
+ String.format("Set oozie admin users to %s", newOozieAdminUsers), "");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/18ba883b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
index 7be5a19..a61b599 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
@@ -147,6 +147,7 @@
<task xsi:type="manual">
<message>Before continuing, please backup the Oozie Server database referenced by the Oozie server located on {{hosts.all}}.</message>
</task>
+ <task xsi:type="server_action" summary="Fix oozie admin users" class="org.apache.ambari.server.serveraction.upgrades.FixOozieAdminUsers"/>
</execute-stage>
<execute-stage service="HIVE" component="HIVE_METASTORE" title="Backup Hive Metastore">
http://git-wip-us.apache.org/repos/asf/ambari/blob/18ba883b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
index 7f277fb..4d5fd07 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
@@ -883,6 +883,8 @@
<script>scripts/oozie_server_upgrade.py</script>
<function>upgrade_oozie_database_and_sharelib</function>
</task>
+ <task xsi:type="server_action" summary="Fixing Oozie admin users"
+ class="org.apache.ambari.server.serveraction.upgrades.FixOozieAdminUsers"/>
</pre-upgrade>
<pre-downgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/18ba883b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
index 2daa9b8..065a933 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
@@ -287,9 +287,11 @@
<!--OOZIE-->
<execute-stage service="OOZIE" component="OOZIE_SERVER" title="Apply config changes for Oozie Server">
<task xsi:type="server_action" summary="Adjusting Oozie properties" class="org.apache.ambari.server.serveraction.upgrades.OozieConfigCalculation"/>
+ <task xsi:type="server_action" summary="Fix oozie admin users" class="org.apache.ambari.server.serveraction.upgrades.FixOozieAdminUsers"/>
</execute-stage>
- <!--FALCON-->
+
+ <!--FALCON-->
<execute-stage service="FALCON" component="FALCON_SERVER" title="Apply config changes for Falcon">
<task xsi:type="configure" id="hdp_2_5_0_0_falcon_server_adjust_services_property"/>
</execute-stage>
http://git-wip-us.apache.org/repos/asf/ambari/blob/18ba883b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
index 43c6265..d152f34 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
@@ -862,6 +862,8 @@
<script>scripts/oozie_server_upgrade.py</script>
<function>upgrade_oozie_database_and_sharelib</function>
</task>
+ <task xsi:type="server_action" summary="Fixing Oozie admin users"
+ class="org.apache.ambari.server.serveraction.upgrades.FixOozieAdminUsers"/>
</pre-upgrade>
<pre-downgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/18ba883b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index a7eed00..be2a011 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -36,36 +36,49 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
def recommendOozieConfigurations(self, configurations, clusterData, services, hosts):
super(HDP25StackAdvisor,self).recommendOozieConfigurations(configurations, clusterData, services, hosts)
putOozieEnvProperty = self.putProperty(configurations, "oozie-env", services)
- if not "FALCON_SERVER" in clusterData["components"] :
- print "Falcon is not part of the installation"
- return
if not "oozie-env" in services["configurations"] :
- print "No oozie env in configurations"
+ Logger.info("No oozie configurations available")
return
- if "falcon-env" in services["configurations"] and "falcon_user" \
- in services["configurations"]["falcon-env"]["properties"] :
- falconUser = services["configurations"]["falcon-env"]["properties"]["falcon_user"]
- print "Falcon user from configuration " + falconUser
- else :
- falconUser = 'falcon'
- print "Defaulting falcon user to " + falconUser
+ falconUser = 'falcon'
+
+ if "falcon-env" in services["configurations"] :
+ if "falcon_user" in services["configurations"]["falcon-env"]["properties"] :
+ falconUser = services["configurations"]["falcon-env"]["properties"]["falcon_user"]
+ Logger.info("Falcon user from configuration: %s " % falconUser)
+
+ Logger.info("Falcon user : %s" % falconUser)
+
+ oozieUser = 'oozie'
if "oozie_user" \
in services["configurations"]["oozie-env"]["properties"] :
oozieUser = services["configurations"]["oozie-env"]["properties"]["oozie_user"]
- print "Oozie user from configuration " + oozieUser
+ Logger.info("Oozie user from configuration %s" % oozieUser)
+
+ Logger.info("Oozie user %s" % oozieUser)
+
+ if "oozie_admin_users" \
+ in services["configurations"]["oozie-env"]["properties"] :
+ currentAdminUsers = services["configurations"]["oozie-env"]["properties"]["oozie_admin_users"]
+ Logger.info("Oozie admin users from configuration %s" % currentAdminUsers)
else :
- oozieUser = 'oozie'
- print "Defaulting oozie user to " + oozieUser
+ currentAdminUsers = "{0}, oozie-admin".format(oozieUser)
+ Logger.info("Setting default oozie admin users to %s" % currentAdminUsers)
+
+
+ if falconUser in currentAdminUsers :
+ Logger.info("Falcon user %s already member of oozie admin users " % falconUser)
+ return
+
+ newAdminUsers = "{0},{1}".format(currentAdminUsers, falconUser)
+
+ Logger.info("new oozie admin users : %s" % newAdminUsers)
- newAdminUsers = "{0},oozie-admin,{1}".format(oozieUser, falconUser)
- print "Setting new oozie admin user to " + newAdminUsers
services["forced-configurations"].append({"type" : "oozie-env", "name" : "oozie_admin_users"})
putOozieEnvProperty("oozie_admin_users", newAdminUsers)
-
def createComponentLayoutRecommendations(self, services, hosts):
parentComponentLayoutRecommendations = super(HDP25StackAdvisor, self).createComponentLayoutRecommendations(
services, hosts)
http://git-wip-us.apache.org/repos/asf/ambari/blob/18ba883b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/FixOozieAdminUsersTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/FixOozieAdminUsersTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/FixOozieAdminUsersTest.java
new file mode 100644
index 0000000..314e955
--- /dev/null
+++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/FixOozieAdminUsersTest.java
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.serveraction.upgrades;
+
+import com.google.inject.Injector;
+import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper;
+import org.apache.ambari.server.actionmanager.HostRoleCommand;
+import org.apache.ambari.server.agent.CommandReport;
+import org.apache.ambari.server.agent.ExecutionCommand;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Config;
+import org.apache.ambari.server.state.ConfigImpl;
+import org.easymock.EasyMock;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.lang.reflect.Field;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.easymock.EasyMock.*;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests OozieConfigCalculation logic
+ */
+public class FixOozieAdminUsersTest {
+ private Injector injector;
+ private Clusters clusters;
+ private Cluster cluster;
+ private Field clustersField;
+
+ @Before
+ public void setup() throws Exception {
+ injector = EasyMock.createMock(Injector.class);
+ clusters = EasyMock.createMock(Clusters.class);
+ cluster = EasyMock.createMock(Cluster.class);
+
+
+ Config falconEnvConfig = new ConfigImpl("falcon-env") {
+ Map<String, String> mockProperties = new HashMap<String, String>() {{
+ put("falcon_user", "falcon");
+ }};
+
+ @Override
+ public Map<String, String> getProperties() {
+ return mockProperties;
+ }
+
+ @Override
+ public void setProperties(Map<String, String> properties) {
+ mockProperties.putAll(properties);
+ }
+
+ @Override
+ public void persist(boolean newConfig) {
+ // no-op
+ }
+ };
+ Config oozieEnvConfig = new ConfigImpl("oozie-env") {
+ Map<String, String> mockProperties = new HashMap<String, String>() {{
+ put("oozie_admin_users", "oozie, oozie-admin");
+ }};
+ @Override
+ public Map<String, String> getProperties() {
+ return mockProperties;
+ }
+
+ @Override
+ public void setProperties(Map<String, String> properties) {
+ mockProperties.putAll(properties);
+ }
+
+ @Override
+ public void persist(boolean newConfig) {
+ // no-op
+ }
+ };
+ expect(cluster.getDesiredConfigByType("falcon-env")).andReturn(falconEnvConfig).atLeastOnce();
+ expect(cluster.getDesiredConfigByType("oozie-env")).andReturn(oozieEnvConfig).atLeastOnce();
+
+ expect(clusters.getCluster((String) anyObject())).andReturn(cluster).anyTimes();
+ expect(injector.getInstance(Clusters.class)).andReturn(clusters).atLeastOnce();
+ replay(injector, clusters);
+
+ clustersField = FixOozieAdminUsers.class.getDeclaredField("clusters");
+ clustersField.setAccessible(true);
+
+ }
+
+ /**
+ * Makes sure falcon user is added to oozie admin users
+ * @throws Exception
+ */
+ @Test
+ public void testOozieAdminUserUpdated() throws Exception {
+
+ Map<String, String> commandParams = new HashMap<String, String>();
+ commandParams.put("clusterName", "c1");
+
+ ExecutionCommand executionCommand = new ExecutionCommand();
+ executionCommand.setCommandParams(commandParams);
+ executionCommand.setClusterName("c1");
+
+ HostRoleCommand hrc = EasyMock.createMock(HostRoleCommand.class);
+ expect(hrc.getRequestId()).andReturn(1L).anyTimes();
+ expect(hrc.getStageId()).andReturn(2L).anyTimes();
+ expect(hrc.getExecutionCommandWrapper()).andReturn(new ExecutionCommandWrapper(executionCommand)).anyTimes();
+ replay(cluster, hrc);
+
+ FixOozieAdminUsers action = new FixOozieAdminUsers();
+ clustersField.set(action, clusters);
+
+ action.setExecutionCommand(executionCommand);
+ action.setHostRoleCommand(hrc);
+
+ CommandReport report = action.execute(null);
+ assertNotNull(report);
+
+
+ Cluster c = clusters.getCluster("c1");
+ Config oozieConfig = c.getDesiredConfigByType("oozie-env");
+ Config falconConfig = c.getDesiredConfigByType("falcon-env");
+
+ Map<String, String> oozieConfigMap = oozieConfig.getProperties();
+ Map<String, String> falconConfigMap = falconConfig.getProperties();
+
+ assertTrue(oozieConfigMap.containsKey("oozie_admin_users"));
+ assertTrue(falconConfigMap.containsKey("falcon_user"));
+ String oozieAdminUsers = oozieConfigMap.get("oozie_admin_users");
+ String falconUser = falconConfigMap.get("falcon_user");
+
+ assertTrue(oozieAdminUsers.indexOf(falconUser) != -1);
+
+ }
+
+}