You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2017/07/31 14:10:34 UTC
[19/25] ambari git commit: AMBARI-21577. Hive-Service check failing
in post EU validation (BI-HDP). (swagle)
AMBARI-21577. Hive-Service check failing in post EU validation (BI-HDP). (swagle)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/fb744009
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/fb744009
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/fb744009
Branch: refs/heads/branch-feature-AMBARI-21450
Commit: fb744009e8d6e269e78943750ec09c73f133470d
Parents: b1438ee
Author: Siddharth Wagle <sw...@hortonworks.com>
Authored: Fri Jul 28 08:44:58 2017 -0700
Committer: Siddharth Wagle <sw...@hortonworks.com>
Committed: Fri Jul 28 08:44:58 2017 -0700
----------------------------------------------------------------------
.../upgrades/FixAuthToLocalMappingAction.java | 149 +++++++++++++++++++
.../upgrades/nonrolling-upgrade-to-hdp-2.6.xml | 8 +
.../upgrades/nonrolling-upgrade-to-hdp-2.6.xml | 8 +
.../FixAuthToLocalMappingActionTest.java | 126 ++++++++++++++++
4 files changed, 291 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/fb744009/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FixAuthToLocalMappingAction.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FixAuthToLocalMappingAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FixAuthToLocalMappingAction.java
new file mode 100644
index 0000000..e62f2db
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FixAuthToLocalMappingAction.java
@@ -0,0 +1,149 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.serveraction.upgrades;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.actionmanager.HostRoleStatus;
+import org.apache.ambari.server.agent.CommandReport;
+import org.apache.ambari.server.controller.KerberosHelper;
+import org.apache.ambari.server.serveraction.AbstractServerAction;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Config;
+import org.apache.ambari.server.state.kerberos.KerberosDescriptor;
+import org.apache.commons.collections.MapUtils;
+import org.apache.commons.lang.StringUtils;
+
+import com.google.inject.Inject;
+
+/**
+ * Fixes auth_to_local rules during upgrade from IOP to HDP. An example of
+ * invalid rule introduced by HBASE_REST_SERVER in the IOP stack, set auth to
+ * local mapping for HTTP spnego principal to local hbase user, which needs to
+ * be deleted for HIVE service check to pass.
+ */
+public class FixAuthToLocalMappingAction extends AbstractServerAction {
+
+ private static final String SPNEGO_PRINC_PATTERN = "RULE:\\[2:\\$1@\\$0\\]\\(HTTP@.*\\)s/\\.\\*/.*/\\n";
+ private static final String AMS_HBASE_PATTERN = "RULE:\\[2:\\$1@\\$0\\]\\(amshbase@.*\\)s/\\.\\*/%s/\\n";
+ private static final String ZK_AMS_PATTERN = "RULE:\\[2:\\$1@\\$0\\]\\(zookeeper@.*\\)s/\\.\\*/%s/\\n";
+
+ @Inject
+ private Clusters clusters;
+
+ @Inject
+ private KerberosHelper kerberosHelper;
+
+ @Override
+ public CommandReport execute(ConcurrentMap<String, Object> requestSharedDataContext)
+ throws AmbariException, InterruptedException {
+
+ String clusterName = getExecutionCommand().getClusterName();
+ Cluster cluster = clusters.getCluster(clusterName);
+
+ KerberosDescriptor kd = kerberosHelper.getKerberosDescriptor(cluster);
+ if (kd == null) {
+ return null;
+ }
+
+ Map<String, Set<String>> configProperties = new HashMap<>();
+ for (String property : kd.getAllAuthToLocalProperties()) {
+ if (!StringUtils.isEmpty(property) && property.contains("/")) {
+ String[] propertyParts = property.split("/");
+ if (configProperties.containsKey(propertyParts[0])) {
+ configProperties.get(propertyParts[0]).add(propertyParts[1]);
+ } else {
+ Set<String> properties = new HashSet<>();
+ properties.add(propertyParts[1]);
+ configProperties.put(propertyParts[0], properties);
+ }
+ }
+ }
+
+ String hbaseUser = null;
+ Config hbaseEnv = cluster.getDesiredConfigByType("hbase-env");
+ if (hbaseEnv != null) {
+ Map<String, String> properties = hbaseEnv.getProperties();
+ if (!MapUtils.isEmpty(properties)) {
+ hbaseUser = properties.get("hbase_user");
+ }
+ }
+
+ String amsUser = null;
+ Config amsEnv = cluster.getDesiredConfigByType("ams-env");
+ if (amsEnv != null) {
+ Map<String, String> properties = amsEnv.getProperties();
+ if (!MapUtils.isEmpty(properties)) {
+ amsUser = properties.get("ambari_metrics_user");
+ }
+ }
+
+ boolean replaced = false;
+ StringBuilder message = new StringBuilder("Replaced offending auto_to_local mappings");
+
+ for (Map.Entry<String, Set<String>> configProperty : configProperties.entrySet()) {
+ String configType = configProperty.getKey();
+ Config config = cluster.getDesiredConfigByType(configType);
+
+ if (config == null) {
+ continue;
+ }
+
+ for (String property : configProperty.getValue()) {
+ Map<String, String> properties = config.getProperties();
+ if (!MapUtils.isEmpty(properties) && properties.containsKey(property)) {
+ String authToLocalRules = properties.get(property);
+ if (!StringUtils.isEmpty(authToLocalRules)) {
+ authToLocalRules = authToLocalRules.replaceAll(SPNEGO_PRINC_PATTERN, "");
+ if (hbaseUser != null) {
+ authToLocalRules = authToLocalRules.replaceAll(String.format(AMS_HBASE_PATTERN, hbaseUser), "");
+ }
+ if (amsUser != null) {
+ authToLocalRules = authToLocalRules.replaceAll(String.format(ZK_AMS_PATTERN, amsUser), "");
+ }
+ // Only if something was replaced
+ if (!properties.get(property).equals(authToLocalRules)) {
+ properties.put(property, authToLocalRules);
+ message.append(" , property => ");
+ message.append(property);
+ message.append(" , config => ");
+ message.append(configType);
+
+ config.setProperties(properties);
+ config.save();
+ replaced = true;
+ }
+ }
+ }
+ }
+ }
+
+ String finalMessage = message.toString();
+ if (!replaced) {
+ finalMessage = "No offending auto_to_local mappings found";
+ }
+
+ return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", finalMessage, "");
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/fb744009/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
index 9d3cc1e..a7fdeec 100644
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
@@ -222,6 +222,14 @@
</task>
</execute-stage>
+ <!-- core-site -->
+ <execute-stage title="Fix auth_to_local mapping">
+ <condition xsi:type="security" type="kerberos"/>
+ <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FixAuthToLocalMappingAction">
+ <summary>Fix auth to local mapping rules</summary>
+ </task>
+ </execute-stage>
+
<!-- YARN -->
<execute-stage service="YARN" component="RESOURCEMANAGER" title="Update Yarn configurations">
<task xsi:type="configure" id="biginsights_4_2_yarn_config_update" />
http://git-wip-us.apache.org/repos/asf/ambari/blob/fb744009/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
index 3637927..5532248 100644
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
@@ -217,6 +217,14 @@
</task>
</execute-stage>
+ <!-- core-site -->
+ <execute-stage title="Fix auth_to_local mapping">
+ <condition xsi:type="security" type="kerberos"/>
+ <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FixAuthToLocalMappingAction">
+ <summary>Fix auth to local mapping rules</summary>
+ </task>
+ </execute-stage>
+
<!-- YARN -->
<execute-stage service="YARN" component="RESOURCEMANAGER" title="Update Yarn configurations">
<task xsi:type="configure" id="biginsights_4_2_yarn_config_update" />
http://git-wip-us.apache.org/repos/asf/ambari/blob/fb744009/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/FixAuthToLocalMappingActionTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/FixAuthToLocalMappingActionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/FixAuthToLocalMappingActionTest.java
new file mode 100644
index 0000000..ab102bc
--- /dev/null
+++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/FixAuthToLocalMappingActionTest.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.serveraction.upgrades;
+
+import static org.easymock.EasyMock.capture;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
+import static org.powermock.api.easymock.PowerMock.verifyAll;
+
+import java.lang.reflect.Field;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper;
+import org.apache.ambari.server.actionmanager.HostRoleCommand;
+import org.apache.ambari.server.agent.ExecutionCommand;
+import org.apache.ambari.server.controller.KerberosHelper;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Config;
+import org.apache.ambari.server.state.kerberos.KerberosDescriptor;
+import org.easymock.Capture;
+import org.junit.Test;
+
+import com.google.common.collect.Maps;
+import com.google.inject.Injector;
+
+import junit.framework.Assert;
+
+public class FixAuthToLocalMappingActionTest {
+
+ String authToLocalRulesOriginal = "RULE:[1:$1@$0](ambari-qa-c1@EXAMPLE.COM)s/.*/ambari-qa/\nRULE:[1:$1@$0](hbase-c1@EXAMPLE.COM)s/.*/hbase/\nRULE:[1:$1@$0](hdfs-c1@EXAMPLE.COM)s/.*/hdfs/\nRULE:[1:$1@$0](.*@EXAMPLE.COM)s/@.*//\nRULE:[2:$1@$0](HTTP@EXAMPLE.COM)s/.*/hbase/\nRULE:[2:$1@$0](amshbase@EXAMPLE.COM)s/.*/ams/\nRULE:[2:$1@$0](dn@EXAMPLE.COM)s/.*/hdfs/\nRULE:[2:$1@$0](hbase@EXAMPLE.COM)s/.*/hbase/\nRULE:[2:$1@$0](hive@EXAMPLE.COM)s/.*/hive/\nRULE:[2:$1@$0](jhs@EXAMPLE.COM)s/.*/mapred/\nRULE:[2:$1@$0](nm@EXAMPLE.COM)s/.*/yarn/\nRULE:[2:$1@$0](nn@EXAMPLE.COM)s/.*/hdfs/\nRULE:[2:$1@$0](rm@EXAMPLE.COM)s/.*/yarn/\nRULE:[2:$1@$0](yarn@EXAMPLE.COM)s/.*/yarn/\nRULE:[2:$1@$0](zookeeper@EXAMPLE.COM)s/.*/ams/\nRULE:[2:$1@$0]([nd]n@.*)s/.*/hdfs/\nRULE:[2:$1@$0]([rn]m@.*)s/.*/yarn/\nRULE:[2:$1@$0](hm@.*)s/.*/hbase/\nRULE:[2:$1@$0](jhs@.*)s/.*/mapred/\nRULE:[2:$1@$0](rs@.*)s/.*/hbase/\nDEFAULT";
+ String authToLocalRulesUpdated = "RULE:[1:$1@$0](ambari-qa-c1@EXAMPLE.COM)s/.*/ambari-qa/\nRULE:[1:$1@$0](hbase-c1@EXAMPLE.COM)s/.*/hbase/\nRULE:[1:$1@$0](hdfs-c1@EXAMPLE.COM)s/.*/hdfs/\nRULE:[1:$1@$0](.*@EXAMPLE.COM)s/@.*//\nRULE:[2:$1@$0](amshbase@EXAMPLE.COM)s/.*/ams/\nRULE:[2:$1@$0](dn@EXAMPLE.COM)s/.*/hdfs/\nRULE:[2:$1@$0](hbase@EXAMPLE.COM)s/.*/hbase/\nRULE:[2:$1@$0](hive@EXAMPLE.COM)s/.*/hive/\nRULE:[2:$1@$0](jhs@EXAMPLE.COM)s/.*/mapred/\nRULE:[2:$1@$0](nm@EXAMPLE.COM)s/.*/yarn/\nRULE:[2:$1@$0](nn@EXAMPLE.COM)s/.*/hdfs/\nRULE:[2:$1@$0](rm@EXAMPLE.COM)s/.*/yarn/\nRULE:[2:$1@$0](yarn@EXAMPLE.COM)s/.*/yarn/\nRULE:[2:$1@$0]([nd]n@.*)s/.*/hdfs/\nRULE:[2:$1@$0]([rn]m@.*)s/.*/yarn/\nRULE:[2:$1@$0](hm@.*)s/.*/hbase/\nRULE:[2:$1@$0](jhs@.*)s/.*/mapred/\nRULE:[2:$1@$0](rs@.*)s/.*/hbase/\nDEFAULT";
+
+ @Test
+ public void testExecute() throws Exception {
+ String clusterName = "c1";
+
+ Injector injector = createNiceMock(Injector.class);
+ Clusters clusters = createNiceMock(Clusters.class);
+ Cluster cluster = createNiceMock(Cluster.class);
+ KerberosHelper kerberosHelper = createNiceMock(KerberosHelper.class);
+ KerberosDescriptor descriptor = createNiceMock(KerberosDescriptor.class);
+
+ expect(kerberosHelper.getKerberosDescriptor(cluster)).andReturn(descriptor).anyTimes();
+ Set<String> mappings = new HashSet<>();
+ mappings.add("core-site/hadoop.security.auth_to_local");
+
+ expect(descriptor.getAllAuthToLocalProperties()).andReturn(mappings);
+
+ Map<String, String> commandParams = Maps.newHashMap();
+ commandParams.put("clusterName", clusterName);
+
+ ExecutionCommand executionCommand = new ExecutionCommand();
+ executionCommand.setCommandParams(commandParams);
+ executionCommand.setClusterName(clusterName);
+
+ HostRoleCommand hrc = createNiceMock(HostRoleCommand.class);
+ expect(hrc.getExecutionCommandWrapper()).andReturn(new ExecutionCommandWrapper(executionCommand));
+
+ Config hbaseEnv = createNiceMock(Config.class);
+ expect(cluster.getDesiredConfigByType("hbase-env")).andReturn(hbaseEnv);
+ expect(hbaseEnv.getProperties()).andReturn(Collections.singletonMap("hbase_user", "hbase"));
+
+ Config amsEnv = createNiceMock(Config.class);
+ expect(cluster.getDesiredConfigByType("ams-env")).andReturn(amsEnv);
+ expect(amsEnv.getProperties()).andReturn(Collections.singletonMap("ambari_metrics_user", "ams"));
+
+ Config coreSite = createNiceMock(Config.class);
+ expect(cluster.getDesiredConfigByType("core-site")).andReturn(coreSite);
+
+ Map<String, String> original = Maps.newHashMap();
+ original.put("hadoop.security.auth_to_local", authToLocalRulesOriginal);
+ expect(coreSite.getProperties()).andReturn(original);
+
+ Capture<Map<String, String>> updated = Capture.newInstance();
+ coreSite.setProperties(capture(updated));
+ expectLastCall();
+ coreSite.save();
+ expectLastCall();
+
+ expect(clusters.getCluster(clusterName)).andReturn(cluster).anyTimes();
+ expect(injector.getInstance(Clusters.class)).andReturn(clusters).atLeastOnce();
+
+ FixAuthToLocalMappingAction action = new FixAuthToLocalMappingAction();
+ action.setExecutionCommand(executionCommand);
+ action.setHostRoleCommand(hrc);
+
+ Field clustersField = FixAuthToLocalMappingAction.class.getDeclaredField("clusters");
+ clustersField.setAccessible(true);
+ clustersField.set(action, clusters);
+
+ Field kerberosHelperField = FixAuthToLocalMappingAction.class.getDeclaredField("kerberosHelper");
+ kerberosHelperField.setAccessible(true);
+ kerberosHelperField.set(action, kerberosHelper);
+
+ replay(kerberosHelper, descriptor, injector, clusters, cluster, hrc, hbaseEnv, amsEnv, coreSite);
+
+ ConcurrentMap<String, Object> emptyMap = Maps.newConcurrentMap();
+ action.execute(emptyMap);
+
+ verifyAll();
+
+ Assert.assertEquals(authToLocalRulesUpdated, updated.getValue().get("hadoop.security.auth_to_local"));
+ }
+}