You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by rl...@apache.org on 2018/07/06 18:48:44 UTC
[ambari] branch branch-2.7 updated: [AMBARI-24250] Create
Checkpoint page stuck while Enabling HA on Namenode
This is an automated email from the ASF dual-hosted git repository.
rlevas pushed a commit to branch branch-2.7
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-2.7 by this push:
new b99e2bf [AMBARI-24250] Create Checkpoint page stuck while Enabling HA on Namenode
b99e2bf is described below
commit b99e2bf52cd4933160194a8cc0f25dbd07325a93
Author: Robert Levas <rl...@hortonworks.com>
AuthorDate: Fri Jul 6 09:43:47 2018 -0400
[AMBARI-24250] Create Checkpoint page stuck while Enabling HA on Namenode
---
.../events/publishers/AgentCommandsPublisher.java | 41 +++++++++++++++++++---
.../stageutils/KerberosKeytabController.java | 19 ++++++----
.../ambari/server/agent/TestHeartbeatHandler.java | 18 +++++++++-
3 files changed, 67 insertions(+), 11 deletions(-)
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/events/publishers/AgentCommandsPublisher.java b/ambari-server/src/main/java/org/apache/ambari/server/events/publishers/AgentCommandsPublisher.java
index 68ad652..c64131d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/events/publishers/AgentCommandsPublisher.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/events/publishers/AgentCommandsPublisher.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -28,6 +28,7 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -47,10 +48,13 @@ import org.apache.ambari.server.serveraction.kerberos.KerberosServerAction;
import org.apache.ambari.server.serveraction.kerberos.stageutils.KerberosKeytabController;
import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosKeytab;
import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosPrincipal;
+import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Service;
import org.apache.ambari.server.utils.StageUtils;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.collections.MapUtils;
import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -174,14 +178,15 @@ public class AgentCommandsPublisher {
* @param targetHost a name of the host the relevant command is destined for
* @throws AmbariException
*/
- void injectKeytab(ExecutionCommand ec, String command, String targetHost) throws AmbariException {
+ private void injectKeytab(ExecutionCommand ec, String command, String targetHost) throws AmbariException {
String dataDir = ec.getCommandParams().get(KerberosServerAction.DATA_DIRECTORY);
KerberosServerAction.KerberosCommandParameters kerberosCommandParameters = new KerberosServerAction.KerberosCommandParameters(ec);
if(dataDir != null) {
List<Map<String, String>> kcp = ec.getKerberosCommandParams();
try {
- Set<ResolvedKerberosKeytab> keytabsToInject = kerberosKeytabController.getFilteredKeytabs((Map<String, Collection<String>>)kerberosCommandParameters.getServiceComponentFilter(), kerberosCommandParameters.getHostFilter(), kerberosCommandParameters.getIdentityFilter());
+ Map<String, Collection<String>> serviceComponentFilter = adjustServiceComponentFilter(ec.getClusterName(), kerberosCommandParameters.getServiceComponentFilter());
+ Set<ResolvedKerberosKeytab> keytabsToInject = kerberosKeytabController.getFilteredKeytabs(serviceComponentFilter, kerberosCommandParameters.getHostFilter(), kerberosCommandParameters.getIdentityFilter());
for (ResolvedKerberosKeytab resolvedKeytab : keytabsToInject) {
for(ResolvedKerberosPrincipal resolvedPrincipal: resolvedKeytab.getPrincipals()) {
String hostName = resolvedPrincipal.getHostName();
@@ -210,7 +215,7 @@ public class AgentCommandsPublisher {
keytabMap.put(KerberosIdentityDataFileReader.KEYTAB_FILE_GROUP_ACCESS, resolvedKeytab.getGroupAccess());
BufferedInputStream bufferedIn = new BufferedInputStream(new FileInputStream(keytabFile));
- byte[] keytabContent = null;
+ byte[] keytabContent;
try {
keytabContent = IOUtils.toByteArray(bufferedIn);
} finally {
@@ -252,4 +257,32 @@ public class AgentCommandsPublisher {
ec.setKerberosCommandParams(kcp);
}
}
+
+ private Map<String, Collection<String>> adjustServiceComponentFilter(String clusterName, Map<String, ? extends Collection<String>> serviceComponentFilter) throws AmbariException {
+ Map<String, Collection<String>> adjustedFilter = new HashMap<>();
+ Cluster cluster = clusters.getCluster(clusterName);
+
+ Map<String, Service> installedServices = (cluster == null) ? null : cluster.getServices();
+
+ if(!MapUtils.isEmpty(installedServices)) {
+ if (serviceComponentFilter != null) {
+ // prune off services that are not installed, or considered installed - like AMBARI
+ for(Map.Entry<String, ? extends Collection<String>> entry: serviceComponentFilter.entrySet()) {
+ String serviceName = entry.getKey();
+
+ if(installedServices.containsKey(serviceName)) {
+ adjustedFilter.put(serviceName, entry.getValue());
+ }
+ }
+ } else {
+ // return only the set of installed services
+ for(String serviceName: installedServices.keySet()) {
+ // Add an entry to indicate the service and all of it's components should be considered
+ adjustedFilter.put(serviceName, Collections.singletonList("*"));
+ }
+ }
+ }
+
+ return adjustedFilter;
+ }
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/stageutils/KerberosKeytabController.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/stageutils/KerberosKeytabController.java
index 4993902..0b01fd9 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/stageutils/KerberosKeytabController.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/stageutils/KerberosKeytabController.java
@@ -133,17 +133,23 @@ public class KerberosKeytabController {
Set<String> serviceSet = new HashSet<>();
Set<String> componentSet = new HashSet<>();
Set<String> serviceOnlySet = new HashSet<>();
- serviceSet.addAll(serviceComponentFilter.keySet());
- for (String serviceName : serviceSet) {
- Collection<String> serviceComponents = serviceComponentFilter.get(serviceName);
- if (serviceComponents.contains("*")) { // star means that this is filtered by whole SERVICE
+
+ // Split the filter into a service/component filter or a service-only filter.
+ for(Map.Entry<String, Collection<String>> entry: serviceComponentFilter.entrySet()) {
+ String serviceName = entry.getKey();
+ Collection<String> serviceComponents = entry.getValue();
+
+ if((serviceComponents == null) || serviceComponents.contains("*")) {
serviceOnlySet.add(serviceName);
- serviceSet.remove(serviceName); // remove service from regular
- } else {
+ }
+ else {
+ serviceSet.add(serviceName);
componentSet.addAll(serviceComponents);
}
}
+
List<KerberosKeytabPrincipalDAO.KerberosKeytabPrincipalFilter> result = new ArrayList<>();
+ // Handle the service/component filter
if (serviceSet.size() > 0) {
result.add(new KerberosKeytabPrincipalDAO.KerberosKeytabPrincipalFilter(
null,
@@ -152,6 +158,7 @@ public class KerberosKeytabController {
null
));
}
+ // Handler the service/* filter
if (serviceOnlySet.size() > 0) {
result.add(new KerberosKeytabPrincipalDAO.KerberosKeytabPrincipalFilter(
null,
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java b/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java
index 00566d7..75d5a7a 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java
@@ -1412,6 +1412,12 @@ public class TestHeartbeatHandler {
List<Map<String, String>> kcp;
Map<String, String> properties;
+ Cluster cluster = heartbeatTestHelper.getDummyCluster();
+ Service hdfs = addService(cluster, HDFS);
+ hdfs.addServiceComponent(DATANODE);
+ hdfs.addServiceComponent(NAMENODE);
+ hdfs.addServiceComponent(SECONDARY_NAMENODE);
+
kcp = testInjectKeytabSetKeytab("c6403.ambari.apache.org");
Assert.assertNotNull(kcp);
Assert.assertEquals(1, kcp.size());
@@ -1448,6 +1454,12 @@ public class TestHeartbeatHandler {
@Test
public void testInjectKeytabNotApplicableHost() throws Exception {
+ Cluster cluster = heartbeatTestHelper.getDummyCluster();
+ Service hdfs = addService(cluster, HDFS);
+ hdfs.addServiceComponent(DATANODE);
+ hdfs.addServiceComponent(NAMENODE);
+ hdfs.addServiceComponent(SECONDARY_NAMENODE);
+
List<Map<String, String>> kcp;
kcp = testInjectKeytabSetKeytab("c6401.ambari.apache.org");
Assert.assertNotNull(kcp);
@@ -1469,6 +1481,7 @@ public class TestHeartbeatHandler {
Map<String, String> commandparams = new HashMap<>();
commandparams.put(KerberosServerAction.AUTHENTICATED_USER_NAME, "admin");
executionCommand.setCommandParams(commandparams);
+ executionCommand.setClusterName(DummyCluster);
final HostRoleCommand command = hostRoleCommandFactory.create(DummyHostname1,
Role.DATANODE, null, null);
@@ -1501,6 +1514,7 @@ public class TestHeartbeatHandler {
Map<String, String> commandparams = new HashMap<>();
commandparams.put(KerberosServerAction.AUTHENTICATED_USER_NAME, "admin");
executionCommand.setCommandParams(commandparams);
+ executionCommand.setClusterName(DummyCluster);
final HostRoleCommand command = hostRoleCommandFactory.create(DummyHostname1,
Role.DATANODE, null, null);
@@ -1524,7 +1538,9 @@ public class TestHeartbeatHandler {
private File createTestKeytabData(AgentCommandsPublisher agentCommandsPublisher) throws Exception {
KerberosKeytabController kerberosKeytabControllerMock = createMock(KerberosKeytabController.class);
- expect(kerberosKeytabControllerMock.getFilteredKeytabs(null,null,null)).andReturn(
+ Map<String, Collection<String>> filter = new HashMap<>();
+ filter.put("HDFS", Collections.singletonList("*"));
+ expect(kerberosKeytabControllerMock.getFilteredKeytabs(filter,null,null)).andReturn(
Sets.newHashSet(
new ResolvedKerberosKeytab(
"/etc/security/keytabs/dn.service.keytab",